diff --git a/examples/bnf_example.py b/examples/bnf_example.py index e278fde..a70d8d6 100644 --- a/examples/bnf_example.py +++ b/examples/bnf_example.py @@ -12,13 +12,14 @@ combining radar volumes with ground stations at multiple locations (M1, S4, S20, S30, S40, S13). """ -import radclss import glob -import matplotlib.pyplot as plt import os +import matplotlib.pyplot as plt from dask.distributed import Client, LocalCluster +import radclss + def main(): date = "20250520" @@ -67,7 +68,7 @@ def main(): for vars in my_columns.data_vars: print(vars, my_columns[vars].dtype) - fig, ax = radclss.vis.create_radclss_columns("radclss_example.nc") + fig, _ax = radclss.vis.create_radclss_columns("radclss_example.nc") print(fig) plt.show() diff --git a/src/radclss/config/__init__.py b/src/radclss/config/__init__.py index 645b05f..80e9056 100644 --- a/src/radclss/config/__init__.py +++ b/src/radclss/config/__init__.py @@ -4,16 +4,16 @@ set_output_facility, # noqa set_output_platform, # noqa set_output_level, # noqa -) # noqa +) from .output_config import ( set_output_gate_time_attrs, # noqa set_output_time_offset_attrs, # noqa get_output_config, # noqa -) # noqa +) from .output_config import ( set_output_alt_attrs, # noqa set_output_lat_attrs, # noqa set_output_lon_attrs, # noqa set_output_station_attrs, # noqa -) # noqa +) from .default_config import DEFAULT_NEXRAD_RADARS # noqa diff --git a/src/radclss/config/default_config.py b/src/radclss/config/default_config.py index 4b09ea9..0f86b23 100644 --- a/src/radclss/config/default_config.py +++ b/src/radclss/config/default_config.py @@ -80,7 +80,7 @@ "normalized_coherent_power", "normalized_coherent_power_v", ], - "radar_xsapr": [ + "radar_xsapr": [ "classification_mask", "censor_mask", "uncorrected_copol_correlation_coeff", @@ -96,7 +96,7 @@ "signal_to_noise_ratio_copolar_v", "normalized_coherent_power", "normalized_coherent_power_v", - "radar_echo_classification" + "radar_echo_classification", ], "met": [ "base_time", @@ -244,5 +244,4 @@ def set_discarded_variables(instrument, var_list): var_list : list List of variable names to discard. """ - global DEFAULT_DISCARD_VAR DEFAULT_DISCARD_VAR[instrument] = var_list diff --git a/src/radclss/config/output_config.py b/src/radclss/config/output_config.py index ec1a9fd..56d60c2 100644 --- a/src/radclss/config/output_config.py +++ b/src/radclss/config/output_config.py @@ -10,26 +10,29 @@ ), "description": "Time in Seconds that Cooresponds to the Minimum Height Gate", } -OUTPUT_TIME_OFFSET_ATTRS = dict( - long_name=("Time in Seconds Since Midnight"), - description=( +OUTPUT_TIME_OFFSET_ATTRS = { + "long_name": ("Time in Seconds Since Midnight"), + "description": ( "Time in Seconds Since Midnight that Cooresponds" + "to the Center of Each Height Gate" + "Above the Target Location " ), -) -OUTPUT_STATION_ATTRS = dict( - long_name="Bankhead National Forest AMF-3 In-Situ Ground Observation Station Identifers" -) -OUTPUT_LAT_ATTRS = dict( - long_name="Latitude of BNF AMF-3 Ground Observation Site", units="Degrees North" -) -OUTPUT_LON_ATTRS = dict( - long_name="Longitude of BNF AMF-3 Ground Observation Site", units="Degrees East" -) -OUTPUT_ALT_ATTRS = dict( - long_name="Altitude above mean sea level for each station", units="m" -) +} +OUTPUT_STATION_ATTRS = { + "long_name": "Bankhead National Forest AMF-3 In-Situ Ground Observation Station Identifers" +} +OUTPUT_LAT_ATTRS = { + "long_name": "Latitude of BNF AMF-3 Ground Observation Site", + "units": "Degrees North", +} +OUTPUT_LON_ATTRS = { + "long_name": "Longitude of BNF AMF-3 Ground Observation Site", + "units": "Degrees East", +} +OUTPUT_ALT_ATTRS = { + "long_name": "Altitude above mean sea level for each station", + "units": "m", +} def set_output_site(site): diff --git a/src/radclss/core/__init__.py b/src/radclss/core/__init__.py index 37c88ac..e3a5695 100644 --- a/src/radclss/core/__init__.py +++ b/src/radclss/core/__init__.py @@ -12,6 +12,6 @@ """ -from .radclss_core import radclss # noqa: F401 +from .radclss_core import radclss __all__ = ["radclss"] diff --git a/src/radclss/core/radclss_core.py b/src/radclss/core/radclss_core.py index 4486c65..30bdf83 100644 --- a/src/radclss/core/radclss_core.py +++ b/src/radclss/core/radclss_core.py @@ -1,21 +1,23 @@ import logging import time -import xarray as xr +import traceback + import act import numpy as np import pandas as pd -import traceback - +import xarray as xr +from dask.distributed import Client, as_completed +from ..config.default_config import DEFAULT_DISCARD_VAR +from ..config.output_config import get_output_config from ..util.column_utils import ( - subset_points, - get_nexrad_column, - _prepare_match, _apply_match, + _prepare_match, + get_nexrad_column, + subset_points, ) -from ..config.default_config import DEFAULT_DISCARD_VAR -from ..config.output_config import get_output_config -from dask.distributed import Client, as_completed + +logger = logging.getLogger(__name__) def radclss( @@ -24,13 +26,13 @@ def radclss( time_coords, serial=True, dod_version="1.0", - discard_var={}, + discard_var=None, verbose=False, base_station="M1", current_client=None, nexrad=True, nexrad_site=None, - height_bins=np.arange(500, 8500, 250), + height_bins=None, ): """ Extracted Radar Columns and In-Situ Sensors @@ -71,7 +73,8 @@ def radclss( Option to supply a Data Object Description version to verify standards. If this is an empty string, then the latest version will be used. Default is '1.0'. discard_var : dict, optional - Dictionary containing variables to drop from each datastream. Default is {}. + Dictionary containing variables to drop from each datastream. Default is None, + which uses the default discard variables for each instrument. verbose : bool, optional Option to print additional information during processing. Default is False. base_station : str, optional @@ -86,7 +89,7 @@ def radclss( Set to None to use the default settings in RadCLss. Default is None. height_bins : numpy.ndarray, optional The height bins in meters to provide the column over. - Default is np.arange(500, 8500, 250). + Default is None, which uses np.arange(500, 8500, 250). Returns ------- @@ -94,10 +97,13 @@ def radclss( Daily time-series of extracted columns saved into ARM formatted netCDF files. """ - if discard_var == {}: + if discard_var is None: discard_var = DEFAULT_DISCARD_VAR - if "sonde" not in volumes.keys(): + if height_bins is None: + height_bins = np.arange(500, 8500, 250) + + if "sonde" not in volumes: volumes["sonde"] = None if verbose: @@ -116,7 +122,7 @@ def radclss( print("-" * 80) if "radar" in time_coords: - if time_coords not in volumes.keys(): + if time_coords not in volumes: raise IndexError( f"{time_coords} is not a valid time basis! Please choose a radar or an " + "Interval" @@ -141,7 +147,7 @@ def radclss( raise RuntimeError( "No Dask client found. Please start a Dask client before running in parallel mode." ) - for k in volumes.keys(): + for k in volumes: if "radar" in k: if verbose: print(f"\nProcessing radar: {k}") @@ -167,7 +173,7 @@ def radclss( print( f" Completed {successful_count}/{len(volumes[k])} files..." ) - except Exception: + except Exception: # noqa: BLE001 failed_count += 1 if verbose: print( @@ -179,15 +185,13 @@ def radclss( f" Finished {k}: {successful_count} successful, {failed_count} failed" ) else: - for k in volumes.keys(): + for k in volumes: if "radar" in k: if verbose: print(f"\nProcessing radar: {k}") print(f" Number of files: {len(volumes[k])}") columns[k] = [] - file_count = 0 - for rad in volumes[k]: - file_count += 1 + for file_count, rad in enumerate(volumes[k], 1): if verbose: print( f" [{file_count}/{len(volumes[k])}] Processing: {rad.split('/')[-1]}" @@ -202,7 +206,7 @@ def radclss( ) columns[k].append(result) - except Exception: + except Exception: # noqa: BLE001 result = None if verbose: traceback.print_exc() @@ -233,15 +237,15 @@ def radclss( nexrad_columns = [] min_times = {} max_times = {} - for k in columns.keys(): - if "radar" in k and len(columns[k]) > 0: + for k, v in columns.items(): + if "radar" in k and len(v) > 0: times = np.array( - [x["base_time"].values.flat[0] for x in columns[k] if x is not None] + [x["base_time"].values.flat[0] for x in v if x is not None] ) min_times[k] = np.min(times) max_times[k] = np.max(times) if verbose: - print(f" {k}: {len(columns[k])} columns") + print(f" {k}: {len(v)} columns") print(f" Time range: {min_times[k]} to {max_times[k]}") min_time = min(np.array([x for x in min_times.values()])) @@ -321,13 +325,13 @@ def _get_nexrad_wrapper(time_str): print( f" Completed {successful_count}/{len(time_list)} NEXRAD columns..." ) - except Exception as error: + except Exception: failed_count += 1 if verbose: print( f" ERROR fetching NEXRAD data (total failures: {failed_count})" ) - logging.exception(error) + logger.exception("Error fetching NEXRAD data") if verbose: print( @@ -356,13 +360,13 @@ def _get_nexrad_wrapper(time_str): print( f" Completed {successful_count}/{len(time_list)} NEXRAD columns..." ) - except Exception as error: + except Exception: failed_count += 1 if verbose: print( f" ERROR fetching NEXRAD data (total failures: {failed_count})" ) - logging.exception(error) + logger.exception("Error fetching NEXRAD data") if verbose: valid_nexrad = sum(1 for x in nexrad_columns if x is not None) @@ -387,10 +391,10 @@ def _get_nexrad_wrapper(time_str): # memory at once before being folded into an intermediate result. _CONCAT_CHUNK = 50 ds_concat = {} - for k in columns.keys(): + for k, v in columns.items(): if verbose: print(f" Processing {k}...") - valid = [data for data in columns[k] if data is not None] + valid = [data for data in v if data is not None] # Non-VPT datasets have no 'time' dimension; VPT datasets do. # Expand non-VPT datasets so all entries have a consistent 'time' # coordinate before concatenation. @@ -444,11 +448,11 @@ def _get_nexrad_wrapper(time_str): if verbose: print(f" Reindexing all datasets to {time_coords} time coordinates") print(f" Reference time steps: {len(ds_concat[time_coords]['time'])}") - for k in ds_concat.keys(): - if not k == time_coords: + for k, v in ds_concat.items(): + if k != time_coords: if verbose: print(f" Reindexing {k}...") - ds_concat[k] = ds_concat[k].reindex( + ds_concat[k] = v.reindex( time=ds_concat[time_coords]["time"], method="nearest" ) @@ -462,17 +466,15 @@ def _get_nexrad_wrapper(time_str): if verbose: print(" Reindexing all datasets to NEXRAD time coordinates") print(f" Reference time steps: {len(nexrad_columns['time'])}") - for k in ds_concat.keys(): + for k, v in ds_concat.items(): if verbose: print(f" Reindexing {k}...") - ds_concat[k] = ds_concat[k].reindex( - time=nexrad_columns["time"], method="nearest" - ) + ds_concat[k] = v.reindex(time=nexrad_columns["time"], method="nearest") elif _is_valid_offset(time_coords): if verbose: print(f" Resampling to {time_coords} intervals") - for k in ds_concat.keys(): - ds_concat[k] = ds_concat[k].resample(time=time_coords).mean() + for k, v in ds_concat.items(): + ds_concat[k] = v.resample(time=time_coords).mean() if nexrad: nexrad_columns = nexrad_columns.resample(time=time_coords).mean() @@ -480,14 +482,16 @@ def _get_nexrad_wrapper(time_str): new_coordinates = pd.date_range(min_time, max_time, freq=time_coords) if verbose: print(f" Creating new time grid: {len(new_coordinates)} time steps") - for k in ds_concat.keys(): - ds_concat[k] = ds_concat[k].reindex(time=new_coordinates, method='nearest') - + for k, v in ds_concat.items(): + ds_concat[k] = v.reindex(time=new_coordinates, method="nearest") + if nexrad: - nexrad_columns = nexrad_columns.reindex(time=new_coordinates, method='nearest') + nexrad_columns = nexrad_columns.reindex( + time=new_coordinates, method="nearest" + ) else: - for k in ds_concat.keys(): - ds_concat[k] = ds_concat[k].reindex(time=ds_times, method="nearest") + for k, v in ds_concat.items(): + ds_concat[k] = v.reindex(time=ds_times, method="nearest") if nexrad: nexrad_columns = nexrad_columns.reindex(time=ds_times, method="nearest") @@ -497,68 +501,73 @@ def _get_nexrad_wrapper(time_str): print("STEP 6: Renaming variables and merging datasets") print("=" * 80) - for k in ds_concat.keys(): + for k, v in ds_concat.items(): radar_name = k.split("_")[1:] if verbose: print(f" Renaming {k} variables with prefix: {'_'.join(radar_name)}_") - for var in ds_concat[k].data_vars: - if var not in [ - "time", - "time_offset", - "base_time", - "height", - "lat", - "lon", - "alt", - "latitude", - "longitude", - ]: - if "sonde_" not in var: - ds_concat[k] = ds_concat[k].rename_vars( - {var: f"{radar_name[0]}_{var}"} - ) + for var in v.data_vars: + if ( + var + not in [ + "time", + "time_offset", + "base_time", + "height", + "lat", + "lon", + "alt", + "latitude", + "longitude", + ] + and "sonde_" not in var + ): + v = v.rename_vars({var: f"{radar_name[0]}_{var}"}) + ds_concat[k] = v if nexrad_columns is not None: if verbose: print(" Renaming NEXRAD variables with prefix: nexrad_") for var in nexrad_columns.data_vars: - if var not in [ - "time", - "time_offset", - "base_time", - "height", - "lat", - "lon", - "alt", - "latitude", - "longitude", - ]: - if "sonde_" not in var: - nexrad_columns = nexrad_columns.rename_vars({var: f"nexrad_{var}"}) + if ( + var + not in [ + "time", + "time_offset", + "base_time", + "height", + "lat", + "lon", + "alt", + "latitude", + "longitude", + ] + and "sonde_" not in var + ): + nexrad_columns = nexrad_columns.rename_vars({var: f"nexrad_{var}"}) if verbose: print(f" Merging {len(ds_concat)} radar datasets...") # Drop time_offset since we won't need it until we write the final dataset - for k in ds_concat.keys(): + for k, v in ds_concat.items(): if verbose: print(f" Time arrays from {k}:") - ds_concat[k] = ds_concat[k].drop(["time_offset", "base_time"]) + ds_concat[k] = v.drop(["time_offset", "base_time"]) if nexrad_columns is not None: nexrad_columns = nexrad_columns.drop(["time_offset", "base_time"]) - first_key = list(ds_concat.keys())[0] + first_key = next(iter(ds_concat.keys())) for k in list(ds_concat.keys())[1:]: for var in ds_concat[k].data_vars: if var in ds_concat[first_key].data_vars: if verbose: print(f"Dropping {var} from {k}") ds_concat[k] = ds_concat[k].drop(var) - + if nexrad_columns is not None: for var in nexrad_columns.data_vars: - for k in ds_concat.keys(): - if var in ds_concat[k].data_vars: + for k, v in ds_concat.items(): + if var in v.data_vars: if verbose: print(f"Dropping {var} from nexrad_columns") nexrad_columns = nexrad_columns.drop(var) @@ -619,58 +628,58 @@ def _get_nexrad_wrapper(time_str): print("=" * 80) for var in ds_concat.data_vars: - if var not in ["time", "time_offset", "base_time", "lat", "lon", "alt"]: - if var in ds.data_vars: - if verbose: - print(f"Adding variable to output dataset: {var}") - print( - f"Original dtype: {ds[var].dtype}, New dtype: {ds_concat[var].dtype}" - ) - old_type = ds[var].dtype - - # Assign data and convert to original dtype - ds[var][:] = ds_concat[var][:] - ds[var] = ds[var].astype(old_type) - if "_FillValue" in ds[var].attrs: - if isinstance(ds[var].attrs["_FillValue"], str): - if ds[var].dtype == "float32": - ds[var].attrs["_FillValue"] = np.float32( - ds[var].attrs["_FillValue"] - ) - elif ds[var].dtype == "float64": - ds[var].attrs["_FillValue"] = np.float64( - ds[var].attrs["_FillValue"] - ) - elif ds[var].dtype == "int32": - ds[var].attrs["_FillValue"] = np.int32( - ds[var].attrs["_FillValue"] - ) - elif ds[var].dtype == "int64": - ds[var].attrs["_FillValue"] = np.int64( - ds[var].attrs["_FillValue"] - ) - ds[var] = ds[var].fillna(ds[var].attrs["_FillValue"]).astype(float) - if "missing_value" in ds[var].attrs: - if isinstance(ds[var].attrs["missing_value"], str): - if ds[var].dtype == "float32": - ds[var].attrs["missing_value"] = np.float32( - ds[var].attrs["missing_value"] - ) - elif ds[var].dtype == "float64": - ds[var].attrs["missing_value"] = np.float64( - ds[var].attrs["missing_value"] - ) - elif ds[var].dtype == "int32": - ds[var].attrs["missing_value"] = np.int32( - ds[var].attrs["missing_value"] - ) - elif ds[var].dtype == "int64": - ds[var].attrs["missing_value"] = np.int64( - ds[var].attrs["missing_value"] - ) - ds[var] = ( - ds[var].fillna(ds[var].attrs["missing_value"]).astype(float) - ) + if ( + var not in ["time", "time_offset", "base_time", "lat", "lon", "alt"] + and var in ds.data_vars + ): + if verbose: + print(f"Adding variable to output dataset: {var}") + print( + f"Original dtype: {ds[var].dtype}, New dtype: {ds_concat[var].dtype}" + ) + old_type = ds[var].dtype + + # Assign data and convert to original dtype + ds[var][:] = ds_concat[var][:] + ds[var] = ds[var].astype(old_type) + if "_FillValue" in ds[var].attrs: + if isinstance(ds[var].attrs["_FillValue"], str): + if ds[var].dtype == "float32": + ds[var].attrs["_FillValue"] = np.float32( + ds[var].attrs["_FillValue"] + ) + elif ds[var].dtype == "float64": + ds[var].attrs["_FillValue"] = np.float64( + ds[var].attrs["_FillValue"] + ) + elif ds[var].dtype == "int32": + ds[var].attrs["_FillValue"] = np.int32( + ds[var].attrs["_FillValue"] + ) + elif ds[var].dtype == "int64": + ds[var].attrs["_FillValue"] = np.int64( + ds[var].attrs["_FillValue"] + ) + ds[var] = ds[var].fillna(ds[var].attrs["_FillValue"]).astype(float) + if "missing_value" in ds[var].attrs: + if isinstance(ds[var].attrs["missing_value"], str): + if ds[var].dtype == "float32": + ds[var].attrs["missing_value"] = np.float32( + ds[var].attrs["missing_value"] + ) + elif ds[var].dtype == "float64": + ds[var].attrs["missing_value"] = np.float64( + ds[var].attrs["missing_value"] + ) + elif ds[var].dtype == "int32": + ds[var].attrs["missing_value"] = np.int32( + ds[var].attrs["missing_value"] + ) + elif ds[var].dtype == "int64": + ds[var].attrs["missing_value"] = np.int64( + ds[var].attrs["missing_value"] + ) + ds[var] = ds[var].fillna(ds[var].attrs["missing_value"]).astype(float) # Remove all the unused CMAC variables # Drop duplicate latitude and longitude @@ -721,15 +730,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var[instrument], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix=f"{instrument}_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var[instrument], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": f"{instrument}_", + }, ) ) elif instrument == "met": @@ -737,14 +746,14 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k][0], - site=site, - discard=discard_var["met"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - ), + { + "ground": volumes[k][0], + "site": site, + "discard": discard_var["met"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + }, ) ) elif instrument == "pluvio": @@ -752,14 +761,14 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["pluvio"], - column_time=ds.time, - column_height=ds["height"], - resample="sum", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["pluvio"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "sum", + }, ) ) elif instrument == "ld": @@ -767,15 +776,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["ldquants"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix="ldquants_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["ldquants"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": "ldquants_", + }, ) ) elif instrument == "vd": @@ -783,15 +792,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["vdisquants"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix="vdisquants_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["vdisquants"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": "vdisquants_", + }, ) ) elif instrument == "wxt": @@ -799,15 +808,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["wxt"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix="wxt_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["wxt"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": "wxt_", + }, ) ) elif instrument == "sonde": @@ -815,15 +824,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["sonde"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix="sonde_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["sonde"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": "sonde_", + }, ) ) diff --git a/src/radclss/io/__init__.py b/src/radclss/io/__init__.py index 4a42069..79bf76b 100644 --- a/src/radclss/io/__init__.py +++ b/src/radclss/io/__init__.py @@ -1,3 +1,3 @@ -from .write import write_radclss_output # noqa +from .write import write_radclss_output __all__ = ["write_radclss_output"] diff --git a/src/radclss/util/__init__.py b/src/radclss/util/__init__.py index f9a68d9..9b3baa8 100644 --- a/src/radclss/util/__init__.py +++ b/src/radclss/util/__init__.py @@ -1,7 +1,7 @@ from .column_utils import ( - subset_points, - match_datasets_act, get_nexrad_column, -) # noqa: F401 + match_datasets_act, + subset_points, +) -__all__ = ["subset_points", "match_datasets_act", "get_nexrad_column"] +__all__ = ["get_nexrad_column", "match_datasets_act", "subset_points"] diff --git a/src/radclss/util/column_utils.py b/src/radclss/util/column_utils.py index aee75bc..d16c0f9 100644 --- a/src/radclss/util/column_utils.py +++ b/src/radclss/util/column_utils.py @@ -1,18 +1,19 @@ -import boto3 -import pyart -import act -import numpy as np -import xarray as xr -import pandas as pd import datetime import logging - from datetime import timedelta -from botocore.config import Config + +import act +import boto3 +import numpy as np +import pandas as pd +import pyart +import xarray as xr from botocore import UNSIGNED +from botocore.config import Config -from ..config import DEFAULT_DISCARD_VAR, DEFAULT_NEXRAD_RADARS -from ..config import get_output_config +from ..config import DEFAULT_DISCARD_VAR, DEFAULT_NEXRAD_RADARS, get_output_config + +logger = logging.getLogger(__name__) _sonde_cache = {} _nexrad_cache = {} @@ -28,7 +29,7 @@ def _read_sonde_cached(path, exclude): def _read_nexrad_cache(path): - if path not in _nexrad_cache.keys(): + if path not in _nexrad_cache: raw = pyart.io.read_nexrad_archive(path) _nexrad_cache[path] = raw # Only maintain 15 files in the cache to save memory @@ -42,7 +43,7 @@ def _grab_90_degree_rays(radar): """Special case for column right over the radar in an RHI""" # Get the rays within 0.5 degrees of 90 degrees ray = np.argmin(np.abs(radar.elevation["data"] - 90.0)) - moment = {key: [] for key in radar.fields.keys()} + moment = {key: [] for key in radar.fields} # Determine the center of each gate for the subsetted rays. rhi_z = radar.range["data"] @@ -71,10 +72,10 @@ def _grab_90_degree_rays(radar): ] # Convert the moment dictionary to xarray DataArray. # Apply radar object meta data to DataArray attribute - for key in moment: + for key, value in moment.items(): if key != "height": da = xr.DataArray( - moment[key], coords=dict(height=zgate), name=key, dims=["height"] + value, coords={"height": zgate}, name=key, dims=["height"] ) for tag in da_meta: if tag in radar.fields[key]: @@ -86,7 +87,7 @@ def _grab_90_degree_rays(radar): # if not present within the radar object. da_base = xr.DataArray(base_time, name="base_time") da_offset = xr.DataArray( - combined_time, coords=dict(height=zgate), name="time_offset", dims=["height"] + combined_time, coords={"height": zgate}, name="time_offset", dims=["height"] ) ds_container.append(da_base.to_dataset(name="base_time")) ds_container.append(da_offset.to_dataset(name="time_offset")) @@ -158,7 +159,7 @@ def _vpt_to_column_timeseries(radar, height_bins): data_vars[key] = xr.DataArray(arr, dims=["time", "height"], attrs=attrs) ds = xr.Dataset(data_vars, coords={"height": zgate, "time": abs_times}) - #ds = ds.dropna("height") + # ds = ds.dropna("height") valid_h = np.isfinite(ds["height"]) print(ds["reflectivity"], np.sum(np.isnan(ds["reflectivity"].values))) if int(valid_h.sum()) > 0: @@ -232,8 +233,7 @@ def _vpt_nan_fill(radar, height_bins): attrs=attrs, ) - ds = xr.Dataset(data_vars, - coords={"height": height_bins, "time": abs_times}) + ds = xr.Dataset(data_vars, coords={"height": height_bins, "time": abs_times}) ds = ds.drop_duplicates("time", keep="first") dedup_times_s = ds["time"].values.astype("datetime64[s]") @@ -263,7 +263,7 @@ def get_nexrad_column( rad_time, site, input_site_dict, - height_bins=np.arange(500, 8500, 250), + height_bins=None, nexrad_radar=None, ): """ @@ -282,8 +282,9 @@ def get_nexrad_column( {'site1': [lat1, lon1, alt1], 'site2': [lat2, lon2, alt2], ...} - height_bins: numpy array - The height bins in meters to provide the column over. + height_bins: numpy array or None + The height bins in meters to provide the column over. Defaults to + ``np.arange(500, 8500, 250)`` when None. nexrad_radar: str or None The NEXRAD radar to obtain the column from. Setting to None will use the default setting for the ARM site. @@ -294,8 +295,11 @@ def get_nexrad_column( An xarray dataset containing the matched columns from the NEXRAD data. """ + if height_bins is None: + height_bins = np.arange(500, 8500, 250) + if nexrad_radar is None: - if site.lower() in DEFAULT_NEXRAD_RADARS.keys(): + if site.lower() in DEFAULT_NEXRAD_RADARS: nexrad_radar = DEFAULT_NEXRAD_RADARS[site.lower()] else: raise UserWarning( @@ -303,11 +307,13 @@ def get_nexrad_column( ) return None - lats = list([x[0] for x in input_site_dict.values()]) - lons = list([x[1] for x in input_site_dict.values()]) - site_alt = list([x[2] for x in input_site_dict.values()]) + lats = [x[0] for x in input_site_dict.values()] + lons = [x[1] for x in input_site_dict.values()] + site_alt = [x[2] for x in input_site_dict.values()] sites = list(input_site_dict.keys()) - right_now = datetime.datetime.strptime(rad_time, "%Y-%m-%dT%H:%M:%S") + right_now = datetime.datetime.strptime(rad_time, "%Y-%m-%dT%H:%M:%S").replace( + tzinfo=datetime.timezone.utc + ) yesterday = right_now - timedelta(days=1) year = right_now.year month = right_now.month @@ -333,7 +339,9 @@ def get_nexrad_column( continue else: time_list.append( - datetime.datetime.strptime(name, f"{nexrad_radar}%Y%m%d_%H%M%S_V06") + datetime.datetime.strptime( + name, f"{nexrad_radar}%Y%m%d_%H%M%S_V06" + ).replace(tzinfo=datetime.timezone.utc) ) time_list = np.array(time_list) @@ -366,9 +374,7 @@ def get_nexrad_column( .interp(height=height_bins) ) else: - target_height = xr.DataArray( - height_bins, dims="height", name="height" - ) + target_height = xr.DataArray(height_bins, dims="height", name="height") da = da.reindex(height=target_height) # Add the latitude and longitude of the extracted column @@ -395,7 +401,7 @@ def subset_points( nfile, input_site_dict, sonde=None, - height_bins=np.arange(500, 8500, 250), + height_bins=None, rad_key="radar_csapr2", **kwargs, ): @@ -419,7 +425,8 @@ def subset_points( radar start time will be used. Default is None. height_bins : numpy array, optional Numpy array containing the desired height bins to interpolate - the extracted radar columns to. Default is np.arange(500, 8500, 250). + the extracted radar columns to. Defaults to + ``np.arange(500, 8500, 250)`` when None. rad_key: str The radar key to use for dropping select variables from the column statistics. @@ -432,19 +439,22 @@ def subset_points( Xarray Dataset containing the radar column above a give set of locations """ + if height_bins is None: + height_bins = np.arange(500, 8500, 250) + ds = None # Define the splash locations [lon,lat] - lats = list([x[0] for x in input_site_dict.values()]) - lons = list([x[1] for x in input_site_dict.values()]) - site_alt = list([x[2] for x in input_site_dict.values()]) + lats = [x[0] for x in input_site_dict.values()] + lons = [x[1] for x in input_site_dict.values()] + site_alt = [x[2] for x in input_site_dict.values()] sites = list(input_site_dict.keys()) try: radar = pyart.io.read(nfile, exclude_fields=DEFAULT_DISCARD_VAR[rad_key]) except OSError: - logging.warning( + logger.warning( f"{nfile} failed to open and is possibly corrupt." + "RadCLss will not generate a column for this file." ) @@ -472,14 +482,14 @@ def subset_points( + "." + nfile.split("/")[-1].split(".")[-2], "%Y%m%d.%H%M%S", - ) + ).replace(tzinfo=datetime.timezone.utc) sonde_start = [ datetime.datetime.strptime( xfile.split("/")[-1].split(".")[2] + "-" + xfile.split("/")[-1].split(".")[3], "%Y%m%d-%H%M%S", - ) + ).replace(tzinfo=datetime.timezone.utc) for xfile in sonde ] # difference in time between radar file and each sonde file @@ -495,7 +505,7 @@ def subset_points( z_dict, sonde_dict = pyart.retrieve.map_profile_to_gates( ds_sonde.variables[var], ds_sonde.variables["alt"], radar ) - field_name = list(radar.fields.keys())[0] + field_name = next(iter(radar.fields.keys())) # add the field to the radar file radar.add_field_like( field_name, @@ -547,8 +557,8 @@ def subset_points( if radar.metadata["facility_id"] == site: try: da = _grab_90_degree_rays(radar) - except Exception: - logging.warning( + except Exception: # noqa: BLE001 + logger.warning( f"Failed to grab 90 degree rays for {site} from {nfile}." + "NaNs will be returned for this column." ) @@ -574,15 +584,13 @@ def subset_points( try: da = da_clean.interp(height=height_bins) except pd.errors.InvalidIndexError: - da_clean = da_clean.drop_duplicates( - "height", keep="first" - ) + da_clean = da_clean.drop_duplicates("height", keep="first") da = da_clean.interp(height=height_bins) time_offset = time_offset.drop_duplicates( "height", keep="first" ) interpolated = True - + if not interpolated: target_height = xr.DataArray( height_bins, dims="height", name="height" @@ -691,7 +699,7 @@ def _prepare_match( "Invalid resample method. Please choose 'mean', 'sum', or 'skip'." ) - matched = matched.assign_coords(coords=dict(station=site)) + matched = matched.assign_coords(coords={"station": site}) matched = matched.expand_dims("station") for attr in ("lat", "lon", "alt"): diff --git a/src/radclss/vis/__init__.py b/src/radclss/vis/__init__.py index 3ac8641..a56259f 100644 --- a/src/radclss/vis/__init__.py +++ b/src/radclss/vis/__init__.py @@ -1,7 +1,7 @@ from .quicklooks import ( - create_radclss_columns, # noqa - create_radclss_rainfall_timeseries, # noqa -) # noqa + create_radclss_columns, + create_radclss_rainfall_timeseries, +) __all__ = [ "create_radclss_columns", diff --git a/src/radclss/vis/quicklooks.py b/src/radclss/vis/quicklooks.py index 3e7e8df..7472b09 100644 --- a/src/radclss/vis/quicklooks.py +++ b/src/radclss/vis/quicklooks.py @@ -1,11 +1,11 @@ -import act -import sys import datetime +import sys +from datetime import timedelta + +import act +import matplotlib.pyplot as plt import numpy as np import xarray as xr -import matplotlib.pyplot as plt - -from datetime import timedelta from mpl_toolkits.axes_grid1 import make_axes_locatable @@ -57,8 +57,8 @@ def create_radclss_columns( if isinstance(radclss, str): try: ds = xr.open_dataset(radclss, decode_timedelta=False) - except Exception as e: - exc_type, exc_obj, exc_tb = sys.exc_info() + except Exception as e: # noqa: BLE001 + _exc_type, _exc_obj, exc_tb = sys.exc_info() # 'e' will contain the error object print( "\nERROR - (create_radclss_timeseries):" @@ -88,7 +88,7 @@ def create_radclss_columns( # Define the time of the radar file we are plotting against radar_time = datetime.datetime.strptime( np.datetime_as_string(ds["time"].data[0], unit="s"), "%Y-%m-%dT%H:%M:%S" - ) + ).replace(tzinfo=datetime.timezone.utc) final_time = radar_time + timedelta(days=1) for i, station in enumerate(stations): row = i // 2 @@ -180,8 +180,8 @@ def create_radclss_rainfall_timeseries( ds = xr.open_dataset(radclss, decode_timedelta=False) elif isinstance(radclss, xr.Dataset): ds = radclss - except Exception as e: - exc_type, exc_obj, exc_tb = sys.exc_info() + except Exception as e: # noqa: BLE001 + _exc_type, _exc_obj, exc_tb = sys.exc_info() # 'e' will contain the error object print( "\nERROR - (create_radclss_timeseries):" @@ -196,7 +196,7 @@ def create_radclss_rainfall_timeseries( # Define the time of the radar file we are plotting against radar_time = datetime.datetime.strptime( np.datetime_as_string(ds["time"].data[0], unit="s"), "%Y-%m-%dT%H:%M:%S" - ) + ).replace(tzinfo=datetime.timezone.utc) final_time = radar_time + timedelta(days=1) # ----------------------------------------------- diff --git a/tests/test_column_utils.py b/tests/test_column_utils.py index f644801..44dc56e 100644 --- a/tests/test_column_utils.py +++ b/tests/test_column_utils.py @@ -1,6 +1,8 @@ +from unittest.mock import MagicMock, patch + import numpy as np import xarray as xr -from unittest.mock import patch, MagicMock + from radclss.util.column_utils import get_nexrad_column diff --git a/tests/test_io.py b/tests/test_io.py index 135c4da..78ea616 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,6 +1,7 @@ -import radclss -import xarray as xr import arm_test_data +import xarray as xr + +import radclss def test_write(): diff --git a/tests/test_radclss.py b/tests/test_radclss.py index 6de8b2c..2f8a17a 100644 --- a/tests/test_radclss.py +++ b/tests/test_radclss.py @@ -1,13 +1,14 @@ -import radclss -import arm_test_data -import os import glob -import xarray as xr +import os + import act +import arm_test_data import numpy as np - +import xarray as xr from distributed import Client, LocalCluster +import radclss + def test_radclss_serial(): test_data_path = arm_test_data.DATASETS.abspath @@ -36,10 +37,9 @@ def test_radclss_serial(): "2025-06-20T00:00:00", output=test_data_path, ) - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) rad_path = os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc") radar_files = sorted(glob.glob(rad_path)) @@ -214,10 +214,9 @@ def test_radclss_parallel(): "2025-06-20T00:00:00", output=test_data_path, ) - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) rad_path = os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc") radar_files = sorted(glob.glob(rad_path)) @@ -473,10 +472,9 @@ def test_radclss_with_kasacr(): ) # Fetch any other test data - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) # Gather all the radar files csapr2_files = sorted( @@ -618,10 +616,9 @@ def test_radclss_with_kazr(): ) # Fetch any other test data - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) # Gather all the radar files csapr2_files = sorted( @@ -856,10 +853,9 @@ def test_radclss_parallel_with_kasacr(): ) # Fetch any other test data - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) # Gather all the radar files csapr2_files = sorted( @@ -959,10 +955,9 @@ def test_match_datasets_act(): radclss_file = arm_test_data.DATASETS.fetch( "bnfcsapr2radclss.c2.20250619.000000.nc" ) - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) met_M1_files = glob.glob(os.path.join(test_data_path, "*bnfmetM1.b1*")) radclss_ds = xr.open_dataset(radclss_file) diff --git a/tests/test_vis.py b/tests/test_vis.py index 75d575f..b05768b 100644 --- a/tests/test_vis.py +++ b/tests/test_vis.py @@ -1,8 +1,9 @@ -import pytest -import radclss import arm_test_data +import pytest import xarray as xr +import radclss + @pytest.mark.mpl_image_compare def test_create_radclss_columns():