From 2068bdddf4f8c39acf915691a4ea1ae6b3fab7bd Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 28 Jul 2026 07:34:27 -0500 Subject: [PATCH] Fix ruff lint errors across the codebase Resolves C408, SIM118, SIM102, PLC0206, DTZ007, C411, RUF059, BLE001, LOG015, B008, RUF015, TRY401, B006, PLW0602, SIM113, and SIM201 violations flagged by ruff. No behavior changes intended. Co-Authored-By: Claude Sonnet 5 --- examples/bnf_example.py | 7 +- src/radclss/config/__init__.py | 6 +- src/radclss/config/default_config.py | 5 +- src/radclss/config/output_config.py | 35 +-- src/radclss/core/__init__.py | 2 +- src/radclss/core/radclss_core.py | 403 ++++++++++++++------------- src/radclss/io/__init__.py | 2 +- src/radclss/util/__init__.py | 8 +- src/radclss/util/column_utils.py | 100 ++++--- src/radclss/vis/__init__.py | 6 +- src/radclss/vis/quicklooks.py | 22 +- tests/test_column_utils.py | 4 +- tests/test_io.py | 5 +- tests/test_radclss.py | 53 ++-- tests/test_vis.py | 5 +- 15 files changed, 341 insertions(+), 322 deletions(-) diff --git a/examples/bnf_example.py b/examples/bnf_example.py index e278fde..a70d8d6 100644 --- a/examples/bnf_example.py +++ b/examples/bnf_example.py @@ -12,13 +12,14 @@ combining radar volumes with ground stations at multiple locations (M1, S4, S20, S30, S40, S13). """ -import radclss import glob -import matplotlib.pyplot as plt import os +import matplotlib.pyplot as plt from dask.distributed import Client, LocalCluster +import radclss + def main(): date = "20250520" @@ -67,7 +68,7 @@ def main(): for vars in my_columns.data_vars: print(vars, my_columns[vars].dtype) - fig, ax = radclss.vis.create_radclss_columns("radclss_example.nc") + fig, _ax = radclss.vis.create_radclss_columns("radclss_example.nc") print(fig) plt.show() diff --git a/src/radclss/config/__init__.py b/src/radclss/config/__init__.py index 645b05f..80e9056 100644 --- a/src/radclss/config/__init__.py +++ b/src/radclss/config/__init__.py @@ -4,16 +4,16 @@ set_output_facility, # noqa set_output_platform, # noqa set_output_level, # noqa -) # noqa +) from .output_config import ( set_output_gate_time_attrs, # noqa set_output_time_offset_attrs, # noqa get_output_config, # noqa -) # noqa +) from .output_config import ( set_output_alt_attrs, # noqa set_output_lat_attrs, # noqa set_output_lon_attrs, # noqa set_output_station_attrs, # noqa -) # noqa +) from .default_config import DEFAULT_NEXRAD_RADARS # noqa diff --git a/src/radclss/config/default_config.py b/src/radclss/config/default_config.py index 4b09ea9..0f86b23 100644 --- a/src/radclss/config/default_config.py +++ b/src/radclss/config/default_config.py @@ -80,7 +80,7 @@ "normalized_coherent_power", "normalized_coherent_power_v", ], - "radar_xsapr": [ + "radar_xsapr": [ "classification_mask", "censor_mask", "uncorrected_copol_correlation_coeff", @@ -96,7 +96,7 @@ "signal_to_noise_ratio_copolar_v", "normalized_coherent_power", "normalized_coherent_power_v", - "radar_echo_classification" + "radar_echo_classification", ], "met": [ "base_time", @@ -244,5 +244,4 @@ def set_discarded_variables(instrument, var_list): var_list : list List of variable names to discard. """ - global DEFAULT_DISCARD_VAR DEFAULT_DISCARD_VAR[instrument] = var_list diff --git a/src/radclss/config/output_config.py b/src/radclss/config/output_config.py index ec1a9fd..56d60c2 100644 --- a/src/radclss/config/output_config.py +++ b/src/radclss/config/output_config.py @@ -10,26 +10,29 @@ ), "description": "Time in Seconds that Cooresponds to the Minimum Height Gate", } -OUTPUT_TIME_OFFSET_ATTRS = dict( - long_name=("Time in Seconds Since Midnight"), - description=( +OUTPUT_TIME_OFFSET_ATTRS = { + "long_name": ("Time in Seconds Since Midnight"), + "description": ( "Time in Seconds Since Midnight that Cooresponds" + "to the Center of Each Height Gate" + "Above the Target Location " ), -) -OUTPUT_STATION_ATTRS = dict( - long_name="Bankhead National Forest AMF-3 In-Situ Ground Observation Station Identifers" -) -OUTPUT_LAT_ATTRS = dict( - long_name="Latitude of BNF AMF-3 Ground Observation Site", units="Degrees North" -) -OUTPUT_LON_ATTRS = dict( - long_name="Longitude of BNF AMF-3 Ground Observation Site", units="Degrees East" -) -OUTPUT_ALT_ATTRS = dict( - long_name="Altitude above mean sea level for each station", units="m" -) +} +OUTPUT_STATION_ATTRS = { + "long_name": "Bankhead National Forest AMF-3 In-Situ Ground Observation Station Identifers" +} +OUTPUT_LAT_ATTRS = { + "long_name": "Latitude of BNF AMF-3 Ground Observation Site", + "units": "Degrees North", +} +OUTPUT_LON_ATTRS = { + "long_name": "Longitude of BNF AMF-3 Ground Observation Site", + "units": "Degrees East", +} +OUTPUT_ALT_ATTRS = { + "long_name": "Altitude above mean sea level for each station", + "units": "m", +} def set_output_site(site): diff --git a/src/radclss/core/__init__.py b/src/radclss/core/__init__.py index 37c88ac..e3a5695 100644 --- a/src/radclss/core/__init__.py +++ b/src/radclss/core/__init__.py @@ -12,6 +12,6 @@ """ -from .radclss_core import radclss # noqa: F401 +from .radclss_core import radclss __all__ = ["radclss"] diff --git a/src/radclss/core/radclss_core.py b/src/radclss/core/radclss_core.py index 4486c65..30bdf83 100644 --- a/src/radclss/core/radclss_core.py +++ b/src/radclss/core/radclss_core.py @@ -1,21 +1,23 @@ import logging import time -import xarray as xr +import traceback + import act import numpy as np import pandas as pd -import traceback - +import xarray as xr +from dask.distributed import Client, as_completed +from ..config.default_config import DEFAULT_DISCARD_VAR +from ..config.output_config import get_output_config from ..util.column_utils import ( - subset_points, - get_nexrad_column, - _prepare_match, _apply_match, + _prepare_match, + get_nexrad_column, + subset_points, ) -from ..config.default_config import DEFAULT_DISCARD_VAR -from ..config.output_config import get_output_config -from dask.distributed import Client, as_completed + +logger = logging.getLogger(__name__) def radclss( @@ -24,13 +26,13 @@ def radclss( time_coords, serial=True, dod_version="1.0", - discard_var={}, + discard_var=None, verbose=False, base_station="M1", current_client=None, nexrad=True, nexrad_site=None, - height_bins=np.arange(500, 8500, 250), + height_bins=None, ): """ Extracted Radar Columns and In-Situ Sensors @@ -71,7 +73,8 @@ def radclss( Option to supply a Data Object Description version to verify standards. If this is an empty string, then the latest version will be used. Default is '1.0'. discard_var : dict, optional - Dictionary containing variables to drop from each datastream. Default is {}. + Dictionary containing variables to drop from each datastream. Default is None, + which uses the default discard variables for each instrument. verbose : bool, optional Option to print additional information during processing. Default is False. base_station : str, optional @@ -86,7 +89,7 @@ def radclss( Set to None to use the default settings in RadCLss. Default is None. height_bins : numpy.ndarray, optional The height bins in meters to provide the column over. - Default is np.arange(500, 8500, 250). + Default is None, which uses np.arange(500, 8500, 250). Returns ------- @@ -94,10 +97,13 @@ def radclss( Daily time-series of extracted columns saved into ARM formatted netCDF files. """ - if discard_var == {}: + if discard_var is None: discard_var = DEFAULT_DISCARD_VAR - if "sonde" not in volumes.keys(): + if height_bins is None: + height_bins = np.arange(500, 8500, 250) + + if "sonde" not in volumes: volumes["sonde"] = None if verbose: @@ -116,7 +122,7 @@ def radclss( print("-" * 80) if "radar" in time_coords: - if time_coords not in volumes.keys(): + if time_coords not in volumes: raise IndexError( f"{time_coords} is not a valid time basis! Please choose a radar or an " + "Interval" @@ -141,7 +147,7 @@ def radclss( raise RuntimeError( "No Dask client found. Please start a Dask client before running in parallel mode." ) - for k in volumes.keys(): + for k in volumes: if "radar" in k: if verbose: print(f"\nProcessing radar: {k}") @@ -167,7 +173,7 @@ def radclss( print( f" Completed {successful_count}/{len(volumes[k])} files..." ) - except Exception: + except Exception: # noqa: BLE001 failed_count += 1 if verbose: print( @@ -179,15 +185,13 @@ def radclss( f" Finished {k}: {successful_count} successful, {failed_count} failed" ) else: - for k in volumes.keys(): + for k in volumes: if "radar" in k: if verbose: print(f"\nProcessing radar: {k}") print(f" Number of files: {len(volumes[k])}") columns[k] = [] - file_count = 0 - for rad in volumes[k]: - file_count += 1 + for file_count, rad in enumerate(volumes[k], 1): if verbose: print( f" [{file_count}/{len(volumes[k])}] Processing: {rad.split('/')[-1]}" @@ -202,7 +206,7 @@ def radclss( ) columns[k].append(result) - except Exception: + except Exception: # noqa: BLE001 result = None if verbose: traceback.print_exc() @@ -233,15 +237,15 @@ def radclss( nexrad_columns = [] min_times = {} max_times = {} - for k in columns.keys(): - if "radar" in k and len(columns[k]) > 0: + for k, v in columns.items(): + if "radar" in k and len(v) > 0: times = np.array( - [x["base_time"].values.flat[0] for x in columns[k] if x is not None] + [x["base_time"].values.flat[0] for x in v if x is not None] ) min_times[k] = np.min(times) max_times[k] = np.max(times) if verbose: - print(f" {k}: {len(columns[k])} columns") + print(f" {k}: {len(v)} columns") print(f" Time range: {min_times[k]} to {max_times[k]}") min_time = min(np.array([x for x in min_times.values()])) @@ -321,13 +325,13 @@ def _get_nexrad_wrapper(time_str): print( f" Completed {successful_count}/{len(time_list)} NEXRAD columns..." ) - except Exception as error: + except Exception: failed_count += 1 if verbose: print( f" ERROR fetching NEXRAD data (total failures: {failed_count})" ) - logging.exception(error) + logger.exception("Error fetching NEXRAD data") if verbose: print( @@ -356,13 +360,13 @@ def _get_nexrad_wrapper(time_str): print( f" Completed {successful_count}/{len(time_list)} NEXRAD columns..." ) - except Exception as error: + except Exception: failed_count += 1 if verbose: print( f" ERROR fetching NEXRAD data (total failures: {failed_count})" ) - logging.exception(error) + logger.exception("Error fetching NEXRAD data") if verbose: valid_nexrad = sum(1 for x in nexrad_columns if x is not None) @@ -387,10 +391,10 @@ def _get_nexrad_wrapper(time_str): # memory at once before being folded into an intermediate result. _CONCAT_CHUNK = 50 ds_concat = {} - for k in columns.keys(): + for k, v in columns.items(): if verbose: print(f" Processing {k}...") - valid = [data for data in columns[k] if data is not None] + valid = [data for data in v if data is not None] # Non-VPT datasets have no 'time' dimension; VPT datasets do. # Expand non-VPT datasets so all entries have a consistent 'time' # coordinate before concatenation. @@ -444,11 +448,11 @@ def _get_nexrad_wrapper(time_str): if verbose: print(f" Reindexing all datasets to {time_coords} time coordinates") print(f" Reference time steps: {len(ds_concat[time_coords]['time'])}") - for k in ds_concat.keys(): - if not k == time_coords: + for k, v in ds_concat.items(): + if k != time_coords: if verbose: print(f" Reindexing {k}...") - ds_concat[k] = ds_concat[k].reindex( + ds_concat[k] = v.reindex( time=ds_concat[time_coords]["time"], method="nearest" ) @@ -462,17 +466,15 @@ def _get_nexrad_wrapper(time_str): if verbose: print(" Reindexing all datasets to NEXRAD time coordinates") print(f" Reference time steps: {len(nexrad_columns['time'])}") - for k in ds_concat.keys(): + for k, v in ds_concat.items(): if verbose: print(f" Reindexing {k}...") - ds_concat[k] = ds_concat[k].reindex( - time=nexrad_columns["time"], method="nearest" - ) + ds_concat[k] = v.reindex(time=nexrad_columns["time"], method="nearest") elif _is_valid_offset(time_coords): if verbose: print(f" Resampling to {time_coords} intervals") - for k in ds_concat.keys(): - ds_concat[k] = ds_concat[k].resample(time=time_coords).mean() + for k, v in ds_concat.items(): + ds_concat[k] = v.resample(time=time_coords).mean() if nexrad: nexrad_columns = nexrad_columns.resample(time=time_coords).mean() @@ -480,14 +482,16 @@ def _get_nexrad_wrapper(time_str): new_coordinates = pd.date_range(min_time, max_time, freq=time_coords) if verbose: print(f" Creating new time grid: {len(new_coordinates)} time steps") - for k in ds_concat.keys(): - ds_concat[k] = ds_concat[k].reindex(time=new_coordinates, method='nearest') - + for k, v in ds_concat.items(): + ds_concat[k] = v.reindex(time=new_coordinates, method="nearest") + if nexrad: - nexrad_columns = nexrad_columns.reindex(time=new_coordinates, method='nearest') + nexrad_columns = nexrad_columns.reindex( + time=new_coordinates, method="nearest" + ) else: - for k in ds_concat.keys(): - ds_concat[k] = ds_concat[k].reindex(time=ds_times, method="nearest") + for k, v in ds_concat.items(): + ds_concat[k] = v.reindex(time=ds_times, method="nearest") if nexrad: nexrad_columns = nexrad_columns.reindex(time=ds_times, method="nearest") @@ -497,68 +501,73 @@ def _get_nexrad_wrapper(time_str): print("STEP 6: Renaming variables and merging datasets") print("=" * 80) - for k in ds_concat.keys(): + for k, v in ds_concat.items(): radar_name = k.split("_")[1:] if verbose: print(f" Renaming {k} variables with prefix: {'_'.join(radar_name)}_") - for var in ds_concat[k].data_vars: - if var not in [ - "time", - "time_offset", - "base_time", - "height", - "lat", - "lon", - "alt", - "latitude", - "longitude", - ]: - if "sonde_" not in var: - ds_concat[k] = ds_concat[k].rename_vars( - {var: f"{radar_name[0]}_{var}"} - ) + for var in v.data_vars: + if ( + var + not in [ + "time", + "time_offset", + "base_time", + "height", + "lat", + "lon", + "alt", + "latitude", + "longitude", + ] + and "sonde_" not in var + ): + v = v.rename_vars({var: f"{radar_name[0]}_{var}"}) + ds_concat[k] = v if nexrad_columns is not None: if verbose: print(" Renaming NEXRAD variables with prefix: nexrad_") for var in nexrad_columns.data_vars: - if var not in [ - "time", - "time_offset", - "base_time", - "height", - "lat", - "lon", - "alt", - "latitude", - "longitude", - ]: - if "sonde_" not in var: - nexrad_columns = nexrad_columns.rename_vars({var: f"nexrad_{var}"}) + if ( + var + not in [ + "time", + "time_offset", + "base_time", + "height", + "lat", + "lon", + "alt", + "latitude", + "longitude", + ] + and "sonde_" not in var + ): + nexrad_columns = nexrad_columns.rename_vars({var: f"nexrad_{var}"}) if verbose: print(f" Merging {len(ds_concat)} radar datasets...") # Drop time_offset since we won't need it until we write the final dataset - for k in ds_concat.keys(): + for k, v in ds_concat.items(): if verbose: print(f" Time arrays from {k}:") - ds_concat[k] = ds_concat[k].drop(["time_offset", "base_time"]) + ds_concat[k] = v.drop(["time_offset", "base_time"]) if nexrad_columns is not None: nexrad_columns = nexrad_columns.drop(["time_offset", "base_time"]) - first_key = list(ds_concat.keys())[0] + first_key = next(iter(ds_concat.keys())) for k in list(ds_concat.keys())[1:]: for var in ds_concat[k].data_vars: if var in ds_concat[first_key].data_vars: if verbose: print(f"Dropping {var} from {k}") ds_concat[k] = ds_concat[k].drop(var) - + if nexrad_columns is not None: for var in nexrad_columns.data_vars: - for k in ds_concat.keys(): - if var in ds_concat[k].data_vars: + for k, v in ds_concat.items(): + if var in v.data_vars: if verbose: print(f"Dropping {var} from nexrad_columns") nexrad_columns = nexrad_columns.drop(var) @@ -619,58 +628,58 @@ def _get_nexrad_wrapper(time_str): print("=" * 80) for var in ds_concat.data_vars: - if var not in ["time", "time_offset", "base_time", "lat", "lon", "alt"]: - if var in ds.data_vars: - if verbose: - print(f"Adding variable to output dataset: {var}") - print( - f"Original dtype: {ds[var].dtype}, New dtype: {ds_concat[var].dtype}" - ) - old_type = ds[var].dtype - - # Assign data and convert to original dtype - ds[var][:] = ds_concat[var][:] - ds[var] = ds[var].astype(old_type) - if "_FillValue" in ds[var].attrs: - if isinstance(ds[var].attrs["_FillValue"], str): - if ds[var].dtype == "float32": - ds[var].attrs["_FillValue"] = np.float32( - ds[var].attrs["_FillValue"] - ) - elif ds[var].dtype == "float64": - ds[var].attrs["_FillValue"] = np.float64( - ds[var].attrs["_FillValue"] - ) - elif ds[var].dtype == "int32": - ds[var].attrs["_FillValue"] = np.int32( - ds[var].attrs["_FillValue"] - ) - elif ds[var].dtype == "int64": - ds[var].attrs["_FillValue"] = np.int64( - ds[var].attrs["_FillValue"] - ) - ds[var] = ds[var].fillna(ds[var].attrs["_FillValue"]).astype(float) - if "missing_value" in ds[var].attrs: - if isinstance(ds[var].attrs["missing_value"], str): - if ds[var].dtype == "float32": - ds[var].attrs["missing_value"] = np.float32( - ds[var].attrs["missing_value"] - ) - elif ds[var].dtype == "float64": - ds[var].attrs["missing_value"] = np.float64( - ds[var].attrs["missing_value"] - ) - elif ds[var].dtype == "int32": - ds[var].attrs["missing_value"] = np.int32( - ds[var].attrs["missing_value"] - ) - elif ds[var].dtype == "int64": - ds[var].attrs["missing_value"] = np.int64( - ds[var].attrs["missing_value"] - ) - ds[var] = ( - ds[var].fillna(ds[var].attrs["missing_value"]).astype(float) - ) + if ( + var not in ["time", "time_offset", "base_time", "lat", "lon", "alt"] + and var in ds.data_vars + ): + if verbose: + print(f"Adding variable to output dataset: {var}") + print( + f"Original dtype: {ds[var].dtype}, New dtype: {ds_concat[var].dtype}" + ) + old_type = ds[var].dtype + + # Assign data and convert to original dtype + ds[var][:] = ds_concat[var][:] + ds[var] = ds[var].astype(old_type) + if "_FillValue" in ds[var].attrs: + if isinstance(ds[var].attrs["_FillValue"], str): + if ds[var].dtype == "float32": + ds[var].attrs["_FillValue"] = np.float32( + ds[var].attrs["_FillValue"] + ) + elif ds[var].dtype == "float64": + ds[var].attrs["_FillValue"] = np.float64( + ds[var].attrs["_FillValue"] + ) + elif ds[var].dtype == "int32": + ds[var].attrs["_FillValue"] = np.int32( + ds[var].attrs["_FillValue"] + ) + elif ds[var].dtype == "int64": + ds[var].attrs["_FillValue"] = np.int64( + ds[var].attrs["_FillValue"] + ) + ds[var] = ds[var].fillna(ds[var].attrs["_FillValue"]).astype(float) + if "missing_value" in ds[var].attrs: + if isinstance(ds[var].attrs["missing_value"], str): + if ds[var].dtype == "float32": + ds[var].attrs["missing_value"] = np.float32( + ds[var].attrs["missing_value"] + ) + elif ds[var].dtype == "float64": + ds[var].attrs["missing_value"] = np.float64( + ds[var].attrs["missing_value"] + ) + elif ds[var].dtype == "int32": + ds[var].attrs["missing_value"] = np.int32( + ds[var].attrs["missing_value"] + ) + elif ds[var].dtype == "int64": + ds[var].attrs["missing_value"] = np.int64( + ds[var].attrs["missing_value"] + ) + ds[var] = ds[var].fillna(ds[var].attrs["missing_value"]).astype(float) # Remove all the unused CMAC variables # Drop duplicate latitude and longitude @@ -721,15 +730,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var[instrument], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix=f"{instrument}_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var[instrument], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": f"{instrument}_", + }, ) ) elif instrument == "met": @@ -737,14 +746,14 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k][0], - site=site, - discard=discard_var["met"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - ), + { + "ground": volumes[k][0], + "site": site, + "discard": discard_var["met"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + }, ) ) elif instrument == "pluvio": @@ -752,14 +761,14 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["pluvio"], - column_time=ds.time, - column_height=ds["height"], - resample="sum", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["pluvio"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "sum", + }, ) ) elif instrument == "ld": @@ -767,15 +776,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["ldquants"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix="ldquants_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["ldquants"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": "ldquants_", + }, ) ) elif instrument == "vd": @@ -783,15 +792,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["vdisquants"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix="vdisquants_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["vdisquants"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": "vdisquants_", + }, ) ) elif instrument == "wxt": @@ -799,15 +808,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["wxt"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix="wxt_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["wxt"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": "wxt_", + }, ) ) elif instrument == "sonde": @@ -815,15 +824,15 @@ def _get_nexrad_wrapper(time_str): ( k, site, - dict( - ground=volumes[k], - site=site, - discard=discard_var["sonde"], - column_time=ds.time, - column_height=ds["height"], - resample="mean", - prefix="sonde_", - ), + { + "ground": volumes[k], + "site": site, + "discard": discard_var["sonde"], + "column_time": ds.time, + "column_height": ds["height"], + "resample": "mean", + "prefix": "sonde_", + }, ) ) diff --git a/src/radclss/io/__init__.py b/src/radclss/io/__init__.py index 4a42069..79bf76b 100644 --- a/src/radclss/io/__init__.py +++ b/src/radclss/io/__init__.py @@ -1,3 +1,3 @@ -from .write import write_radclss_output # noqa +from .write import write_radclss_output __all__ = ["write_radclss_output"] diff --git a/src/radclss/util/__init__.py b/src/radclss/util/__init__.py index f9a68d9..9b3baa8 100644 --- a/src/radclss/util/__init__.py +++ b/src/radclss/util/__init__.py @@ -1,7 +1,7 @@ from .column_utils import ( - subset_points, - match_datasets_act, get_nexrad_column, -) # noqa: F401 + match_datasets_act, + subset_points, +) -__all__ = ["subset_points", "match_datasets_act", "get_nexrad_column"] +__all__ = ["get_nexrad_column", "match_datasets_act", "subset_points"] diff --git a/src/radclss/util/column_utils.py b/src/radclss/util/column_utils.py index aee75bc..d16c0f9 100644 --- a/src/radclss/util/column_utils.py +++ b/src/radclss/util/column_utils.py @@ -1,18 +1,19 @@ -import boto3 -import pyart -import act -import numpy as np -import xarray as xr -import pandas as pd import datetime import logging - from datetime import timedelta -from botocore.config import Config + +import act +import boto3 +import numpy as np +import pandas as pd +import pyart +import xarray as xr from botocore import UNSIGNED +from botocore.config import Config -from ..config import DEFAULT_DISCARD_VAR, DEFAULT_NEXRAD_RADARS -from ..config import get_output_config +from ..config import DEFAULT_DISCARD_VAR, DEFAULT_NEXRAD_RADARS, get_output_config + +logger = logging.getLogger(__name__) _sonde_cache = {} _nexrad_cache = {} @@ -28,7 +29,7 @@ def _read_sonde_cached(path, exclude): def _read_nexrad_cache(path): - if path not in _nexrad_cache.keys(): + if path not in _nexrad_cache: raw = pyart.io.read_nexrad_archive(path) _nexrad_cache[path] = raw # Only maintain 15 files in the cache to save memory @@ -42,7 +43,7 @@ def _grab_90_degree_rays(radar): """Special case for column right over the radar in an RHI""" # Get the rays within 0.5 degrees of 90 degrees ray = np.argmin(np.abs(radar.elevation["data"] - 90.0)) - moment = {key: [] for key in radar.fields.keys()} + moment = {key: [] for key in radar.fields} # Determine the center of each gate for the subsetted rays. rhi_z = radar.range["data"] @@ -71,10 +72,10 @@ def _grab_90_degree_rays(radar): ] # Convert the moment dictionary to xarray DataArray. # Apply radar object meta data to DataArray attribute - for key in moment: + for key, value in moment.items(): if key != "height": da = xr.DataArray( - moment[key], coords=dict(height=zgate), name=key, dims=["height"] + value, coords={"height": zgate}, name=key, dims=["height"] ) for tag in da_meta: if tag in radar.fields[key]: @@ -86,7 +87,7 @@ def _grab_90_degree_rays(radar): # if not present within the radar object. da_base = xr.DataArray(base_time, name="base_time") da_offset = xr.DataArray( - combined_time, coords=dict(height=zgate), name="time_offset", dims=["height"] + combined_time, coords={"height": zgate}, name="time_offset", dims=["height"] ) ds_container.append(da_base.to_dataset(name="base_time")) ds_container.append(da_offset.to_dataset(name="time_offset")) @@ -158,7 +159,7 @@ def _vpt_to_column_timeseries(radar, height_bins): data_vars[key] = xr.DataArray(arr, dims=["time", "height"], attrs=attrs) ds = xr.Dataset(data_vars, coords={"height": zgate, "time": abs_times}) - #ds = ds.dropna("height") + # ds = ds.dropna("height") valid_h = np.isfinite(ds["height"]) print(ds["reflectivity"], np.sum(np.isnan(ds["reflectivity"].values))) if int(valid_h.sum()) > 0: @@ -232,8 +233,7 @@ def _vpt_nan_fill(radar, height_bins): attrs=attrs, ) - ds = xr.Dataset(data_vars, - coords={"height": height_bins, "time": abs_times}) + ds = xr.Dataset(data_vars, coords={"height": height_bins, "time": abs_times}) ds = ds.drop_duplicates("time", keep="first") dedup_times_s = ds["time"].values.astype("datetime64[s]") @@ -263,7 +263,7 @@ def get_nexrad_column( rad_time, site, input_site_dict, - height_bins=np.arange(500, 8500, 250), + height_bins=None, nexrad_radar=None, ): """ @@ -282,8 +282,9 @@ def get_nexrad_column( {'site1': [lat1, lon1, alt1], 'site2': [lat2, lon2, alt2], ...} - height_bins: numpy array - The height bins in meters to provide the column over. + height_bins: numpy array or None + The height bins in meters to provide the column over. Defaults to + ``np.arange(500, 8500, 250)`` when None. nexrad_radar: str or None The NEXRAD radar to obtain the column from. Setting to None will use the default setting for the ARM site. @@ -294,8 +295,11 @@ def get_nexrad_column( An xarray dataset containing the matched columns from the NEXRAD data. """ + if height_bins is None: + height_bins = np.arange(500, 8500, 250) + if nexrad_radar is None: - if site.lower() in DEFAULT_NEXRAD_RADARS.keys(): + if site.lower() in DEFAULT_NEXRAD_RADARS: nexrad_radar = DEFAULT_NEXRAD_RADARS[site.lower()] else: raise UserWarning( @@ -303,11 +307,13 @@ def get_nexrad_column( ) return None - lats = list([x[0] for x in input_site_dict.values()]) - lons = list([x[1] for x in input_site_dict.values()]) - site_alt = list([x[2] for x in input_site_dict.values()]) + lats = [x[0] for x in input_site_dict.values()] + lons = [x[1] for x in input_site_dict.values()] + site_alt = [x[2] for x in input_site_dict.values()] sites = list(input_site_dict.keys()) - right_now = datetime.datetime.strptime(rad_time, "%Y-%m-%dT%H:%M:%S") + right_now = datetime.datetime.strptime(rad_time, "%Y-%m-%dT%H:%M:%S").replace( + tzinfo=datetime.timezone.utc + ) yesterday = right_now - timedelta(days=1) year = right_now.year month = right_now.month @@ -333,7 +339,9 @@ def get_nexrad_column( continue else: time_list.append( - datetime.datetime.strptime(name, f"{nexrad_radar}%Y%m%d_%H%M%S_V06") + datetime.datetime.strptime( + name, f"{nexrad_radar}%Y%m%d_%H%M%S_V06" + ).replace(tzinfo=datetime.timezone.utc) ) time_list = np.array(time_list) @@ -366,9 +374,7 @@ def get_nexrad_column( .interp(height=height_bins) ) else: - target_height = xr.DataArray( - height_bins, dims="height", name="height" - ) + target_height = xr.DataArray(height_bins, dims="height", name="height") da = da.reindex(height=target_height) # Add the latitude and longitude of the extracted column @@ -395,7 +401,7 @@ def subset_points( nfile, input_site_dict, sonde=None, - height_bins=np.arange(500, 8500, 250), + height_bins=None, rad_key="radar_csapr2", **kwargs, ): @@ -419,7 +425,8 @@ def subset_points( radar start time will be used. Default is None. height_bins : numpy array, optional Numpy array containing the desired height bins to interpolate - the extracted radar columns to. Default is np.arange(500, 8500, 250). + the extracted radar columns to. Defaults to + ``np.arange(500, 8500, 250)`` when None. rad_key: str The radar key to use for dropping select variables from the column statistics. @@ -432,19 +439,22 @@ def subset_points( Xarray Dataset containing the radar column above a give set of locations """ + if height_bins is None: + height_bins = np.arange(500, 8500, 250) + ds = None # Define the splash locations [lon,lat] - lats = list([x[0] for x in input_site_dict.values()]) - lons = list([x[1] for x in input_site_dict.values()]) - site_alt = list([x[2] for x in input_site_dict.values()]) + lats = [x[0] for x in input_site_dict.values()] + lons = [x[1] for x in input_site_dict.values()] + site_alt = [x[2] for x in input_site_dict.values()] sites = list(input_site_dict.keys()) try: radar = pyart.io.read(nfile, exclude_fields=DEFAULT_DISCARD_VAR[rad_key]) except OSError: - logging.warning( + logger.warning( f"{nfile} failed to open and is possibly corrupt." + "RadCLss will not generate a column for this file." ) @@ -472,14 +482,14 @@ def subset_points( + "." + nfile.split("/")[-1].split(".")[-2], "%Y%m%d.%H%M%S", - ) + ).replace(tzinfo=datetime.timezone.utc) sonde_start = [ datetime.datetime.strptime( xfile.split("/")[-1].split(".")[2] + "-" + xfile.split("/")[-1].split(".")[3], "%Y%m%d-%H%M%S", - ) + ).replace(tzinfo=datetime.timezone.utc) for xfile in sonde ] # difference in time between radar file and each sonde file @@ -495,7 +505,7 @@ def subset_points( z_dict, sonde_dict = pyart.retrieve.map_profile_to_gates( ds_sonde.variables[var], ds_sonde.variables["alt"], radar ) - field_name = list(radar.fields.keys())[0] + field_name = next(iter(radar.fields.keys())) # add the field to the radar file radar.add_field_like( field_name, @@ -547,8 +557,8 @@ def subset_points( if radar.metadata["facility_id"] == site: try: da = _grab_90_degree_rays(radar) - except Exception: - logging.warning( + except Exception: # noqa: BLE001 + logger.warning( f"Failed to grab 90 degree rays for {site} from {nfile}." + "NaNs will be returned for this column." ) @@ -574,15 +584,13 @@ def subset_points( try: da = da_clean.interp(height=height_bins) except pd.errors.InvalidIndexError: - da_clean = da_clean.drop_duplicates( - "height", keep="first" - ) + da_clean = da_clean.drop_duplicates("height", keep="first") da = da_clean.interp(height=height_bins) time_offset = time_offset.drop_duplicates( "height", keep="first" ) interpolated = True - + if not interpolated: target_height = xr.DataArray( height_bins, dims="height", name="height" @@ -691,7 +699,7 @@ def _prepare_match( "Invalid resample method. Please choose 'mean', 'sum', or 'skip'." ) - matched = matched.assign_coords(coords=dict(station=site)) + matched = matched.assign_coords(coords={"station": site}) matched = matched.expand_dims("station") for attr in ("lat", "lon", "alt"): diff --git a/src/radclss/vis/__init__.py b/src/radclss/vis/__init__.py index 3ac8641..a56259f 100644 --- a/src/radclss/vis/__init__.py +++ b/src/radclss/vis/__init__.py @@ -1,7 +1,7 @@ from .quicklooks import ( - create_radclss_columns, # noqa - create_radclss_rainfall_timeseries, # noqa -) # noqa + create_radclss_columns, + create_radclss_rainfall_timeseries, +) __all__ = [ "create_radclss_columns", diff --git a/src/radclss/vis/quicklooks.py b/src/radclss/vis/quicklooks.py index 3e7e8df..7472b09 100644 --- a/src/radclss/vis/quicklooks.py +++ b/src/radclss/vis/quicklooks.py @@ -1,11 +1,11 @@ -import act -import sys import datetime +import sys +from datetime import timedelta + +import act +import matplotlib.pyplot as plt import numpy as np import xarray as xr -import matplotlib.pyplot as plt - -from datetime import timedelta from mpl_toolkits.axes_grid1 import make_axes_locatable @@ -57,8 +57,8 @@ def create_radclss_columns( if isinstance(radclss, str): try: ds = xr.open_dataset(radclss, decode_timedelta=False) - except Exception as e: - exc_type, exc_obj, exc_tb = sys.exc_info() + except Exception as e: # noqa: BLE001 + _exc_type, _exc_obj, exc_tb = sys.exc_info() # 'e' will contain the error object print( "\nERROR - (create_radclss_timeseries):" @@ -88,7 +88,7 @@ def create_radclss_columns( # Define the time of the radar file we are plotting against radar_time = datetime.datetime.strptime( np.datetime_as_string(ds["time"].data[0], unit="s"), "%Y-%m-%dT%H:%M:%S" - ) + ).replace(tzinfo=datetime.timezone.utc) final_time = radar_time + timedelta(days=1) for i, station in enumerate(stations): row = i // 2 @@ -180,8 +180,8 @@ def create_radclss_rainfall_timeseries( ds = xr.open_dataset(radclss, decode_timedelta=False) elif isinstance(radclss, xr.Dataset): ds = radclss - except Exception as e: - exc_type, exc_obj, exc_tb = sys.exc_info() + except Exception as e: # noqa: BLE001 + _exc_type, _exc_obj, exc_tb = sys.exc_info() # 'e' will contain the error object print( "\nERROR - (create_radclss_timeseries):" @@ -196,7 +196,7 @@ def create_radclss_rainfall_timeseries( # Define the time of the radar file we are plotting against radar_time = datetime.datetime.strptime( np.datetime_as_string(ds["time"].data[0], unit="s"), "%Y-%m-%dT%H:%M:%S" - ) + ).replace(tzinfo=datetime.timezone.utc) final_time = radar_time + timedelta(days=1) # ----------------------------------------------- diff --git a/tests/test_column_utils.py b/tests/test_column_utils.py index f644801..44dc56e 100644 --- a/tests/test_column_utils.py +++ b/tests/test_column_utils.py @@ -1,6 +1,8 @@ +from unittest.mock import MagicMock, patch + import numpy as np import xarray as xr -from unittest.mock import patch, MagicMock + from radclss.util.column_utils import get_nexrad_column diff --git a/tests/test_io.py b/tests/test_io.py index 135c4da..78ea616 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,6 +1,7 @@ -import radclss -import xarray as xr import arm_test_data +import xarray as xr + +import radclss def test_write(): diff --git a/tests/test_radclss.py b/tests/test_radclss.py index 6de8b2c..2f8a17a 100644 --- a/tests/test_radclss.py +++ b/tests/test_radclss.py @@ -1,13 +1,14 @@ -import radclss -import arm_test_data -import os import glob -import xarray as xr +import os + import act +import arm_test_data import numpy as np - +import xarray as xr from distributed import Client, LocalCluster +import radclss + def test_radclss_serial(): test_data_path = arm_test_data.DATASETS.abspath @@ -36,10 +37,9 @@ def test_radclss_serial(): "2025-06-20T00:00:00", output=test_data_path, ) - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) rad_path = os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc") radar_files = sorted(glob.glob(rad_path)) @@ -214,10 +214,9 @@ def test_radclss_parallel(): "2025-06-20T00:00:00", output=test_data_path, ) - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) rad_path = os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc") radar_files = sorted(glob.glob(rad_path)) @@ -473,10 +472,9 @@ def test_radclss_with_kasacr(): ) # Fetch any other test data - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) # Gather all the radar files csapr2_files = sorted( @@ -618,10 +616,9 @@ def test_radclss_with_kazr(): ) # Fetch any other test data - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) # Gather all the radar files csapr2_files = sorted( @@ -856,10 +853,9 @@ def test_radclss_parallel_with_kasacr(): ) # Fetch any other test data - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) # Gather all the radar files csapr2_files = sorted( @@ -959,10 +955,9 @@ def test_match_datasets_act(): radclss_file = arm_test_data.DATASETS.fetch( "bnfcsapr2radclss.c2.20250619.000000.nc" ) - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) + for files in arm_test_data.DATASETS.registry: + if "bnf" in files and not os.path.exists(os.path.join(test_data_path, files)): + arm_test_data.DATASETS.fetch(files) met_M1_files = glob.glob(os.path.join(test_data_path, "*bnfmetM1.b1*")) radclss_ds = xr.open_dataset(radclss_file) diff --git a/tests/test_vis.py b/tests/test_vis.py index 75d575f..b05768b 100644 --- a/tests/test_vis.py +++ b/tests/test_vis.py @@ -1,8 +1,9 @@ -import pytest -import radclss import arm_test_data +import pytest import xarray as xr +import radclss + @pytest.mark.mpl_image_compare def test_create_radclss_columns():