From da5d968c45dfa19d56d7374d4c01e08a4e8a2297 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 19 Oct 2026 17:57:18 -0400 Subject: [PATCH 01/24] ENH: added verify flag Added a flag to turn off verify, making downloads more secure by default and still allowing a bypass. --- GeospaceDataManagement/instruments/sw_ap.py | 9 +++++++-- GeospaceDataManagement/instruments/sw_apo.py | 8 ++++++-- GeospaceDataManagement/instruments/sw_f107.py | 8 ++++++-- GeospaceDataManagement/instruments/sw_hpo.py | 8 ++++++-- GeospaceDataManagement/instruments/sw_kp.py | 9 +++++++-- GeospaceDataManagement/instruments/sw_ssn.py | 8 ++++++-- 6 files changed, 38 insertions(+), 12 deletions(-) diff --git a/GeospaceDataManagement/instruments/sw_ap.py b/GeospaceDataManagement/instruments/sw_ap.py index d5cf4f4..6c3c57f 100644 --- a/GeospaceDataManagement/instruments/sw_ap.py +++ b/GeospaceDataManagement/instruments/sw_ap.py @@ -227,7 +227,8 @@ def list_files(tag='', inst_id='', data_path='', format_str=None): return files -def download(date_array, tag, inst_id, data_path, mock_download_dir=None): +def download(date_array, tag, inst_id, data_path, mock_download_dir=None, + verify=True): """Download the Ap index data from the appropriate repository. Parameters @@ -244,6 +245,9 @@ def download(date_array, tag, inst_id, data_path, mock_download_dir=None): Local directory with downloaded files or None. If not None, will process any files with the correct name and date as if they were downloaded (default=None) + verify : bool + If True, uses certificate verification; if False will not do so. Only + used for 'def' and 'now' tags (default=True) Note ---- @@ -262,7 +266,8 @@ def download(date_array, tag, inst_id, data_path, mock_download_dir=None): if tag in ['def', 'now']: sw_methods.gfz.kp_ap_cp_download(platform, name, date_array, tag, - inst_id, data_path, mock_download_dir) + inst_id, data_path, mock_download_dir, + verify=verify) elif tag == 'recent': sw_methods.swpc.kp_ap_recent_download(name, date_array, data_path, mock_download_dir) diff --git a/GeospaceDataManagement/instruments/sw_apo.py b/GeospaceDataManagement/instruments/sw_apo.py index dc220d5..3ef9c55 100644 --- a/GeospaceDataManagement/instruments/sw_apo.py +++ b/GeospaceDataManagement/instruments/sw_apo.py @@ -170,7 +170,7 @@ def list_files(tag='', inst_id='', data_path='', format_str=None): def download(date_array, tag, inst_id, data_path, update_files=False, - mock_download_dir=None): + mock_download_dir=None, verify=True): """Download the apo index data from the appropriate repository. Parameters @@ -189,6 +189,9 @@ def download(date_array, tag, inst_id, data_path, update_files=False, Local directory with downloaded files or None. If not None, will process any files with the correct name and date as if they were downloaded (default=None) + verify : bool + If True, uses certificate verification; if False will not do so. + (default=True) Note ---- @@ -214,6 +217,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False, sw_methods.gfz.json_downloads(date_array, data_path, local_file_prefix, local_date_fmt, gfz_data_name, freq, update_files=update_files, - mock_download_dir=mock_download_dir) + mock_download_dir=mock_download_dir, + verify=verify) return diff --git a/GeospaceDataManagement/instruments/sw_f107.py b/GeospaceDataManagement/instruments/sw_f107.py index 776ac08..01ef603 100644 --- a/GeospaceDataManagement/instruments/sw_f107.py +++ b/GeospaceDataManagement/instruments/sw_f107.py @@ -341,7 +341,7 @@ def list_files(tag='', inst_id='', data_path='', format_str=None): def download(date_array, tag, inst_id, data_path, update_files=False, - mock_download_dir=None): + mock_download_dir=None, verify=True): """Download F107 index data from the appropriate repository. Parameters @@ -360,6 +360,9 @@ def download(date_array, tag, inst_id, data_path, update_files=False, Local directory with downloaded files or None. If not None, will process any files with the correct name and date as if they were downloaded (default=None) + verify : bool + If True, uses certificate verification; if False will not do so. Only + used for the 'now' tag (default=True) Raises ------ @@ -415,7 +418,8 @@ def download(date_array, tag, inst_id, data_path, update_files=False, "%Y-%m", gfz_data_name, pds.DateOffset(months=1, seconds=-1), update_files=update_files, - mock_download_dir=mock_download_dir) + mock_download_dir=mock_download_dir, + verify=verify) elif tag == 'daily': sw_methods.swpc.daily_dsd_download(name, today, data_path, diff --git a/GeospaceDataManagement/instruments/sw_hpo.py b/GeospaceDataManagement/instruments/sw_hpo.py index 7d96614..bd7bbfb 100644 --- a/GeospaceDataManagement/instruments/sw_hpo.py +++ b/GeospaceDataManagement/instruments/sw_hpo.py @@ -170,7 +170,7 @@ def list_files(tag='', inst_id='', data_path='', format_str=None): def download(date_array, tag, inst_id, data_path, update_files=False, - mock_download_dir=None): + mock_download_dir=None, verify=True): """Download the Hpo index data from the appropriate repository. Parameters @@ -189,6 +189,9 @@ def download(date_array, tag, inst_id, data_path, update_files=False, Local directory with downloaded files or None. If not None, will process any files with the correct name and date as if they were downloaded (default=None) + verify : bool + If True, uses certificate verification; if False will not do so + (default=True) Note ---- @@ -215,6 +218,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False, sw_methods.gfz.json_downloads(date_array, data_path, local_file_prefix, local_date_fmt, gfz_data_name, freq, update_files=update_files, - mock_download_dir=mock_download_dir) + mock_download_dir=mock_download_dir, + verify=verify) return diff --git a/GeospaceDataManagement/instruments/sw_kp.py b/GeospaceDataManagement/instruments/sw_kp.py index 9ae1797..e5ca000 100644 --- a/GeospaceDataManagement/instruments/sw_kp.py +++ b/GeospaceDataManagement/instruments/sw_kp.py @@ -218,7 +218,8 @@ def list_files(tag='', inst_id='', data_path='', format_str=None): return files -def download(date_array, tag, inst_id, data_path, mock_download_dir=None): +def download(date_array, tag, inst_id, data_path, mock_download_dir=None, + verify=True): """Download the Kp index data from the appropriate repository. Parameters @@ -235,6 +236,9 @@ def download(date_array, tag, inst_id, data_path, mock_download_dir=None): Local directory with downloaded files or None. If not None, will process any files with the correct name and date as if they were downloaded (default=None) + verify : bool + If True, uses certificate verification; if False will not do so. Only + used for 'def' and 'now' tags (default=True) Raises ------ @@ -255,7 +259,8 @@ def download(date_array, tag, inst_id, data_path, mock_download_dir=None): # Download standard Kp data if tag in ['def', 'now']: sw_methods.gfz.kp_ap_cp_download(platform, name, date_array, tag, - inst_id, data_path, mock_download_dir) + inst_id, data_path, mock_download_dir, + verify=verify) elif tag == 'forecast': sw_methods.swpc.geomag_forecast_download(name, date_array, data_path, mock_download_dir) diff --git a/GeospaceDataManagement/instruments/sw_ssn.py b/GeospaceDataManagement/instruments/sw_ssn.py index 7d74a10..2af87c4 100644 --- a/GeospaceDataManagement/instruments/sw_ssn.py +++ b/GeospaceDataManagement/instruments/sw_ssn.py @@ -262,7 +262,7 @@ def list_files(tag='', inst_id='', data_path='', format_str=None): def download(date_array, tag, inst_id, data_path, update_files=False, - mock_download_dir=None): + mock_download_dir=None, verify=True): """Download solar magnetic field data from the appropriate repository. Parameters @@ -281,6 +281,9 @@ def download(date_array, tag, inst_id, data_path, update_files=False, Local directory with downloaded files or None. If not None, will process any files with the correct name and date as if they were downloaded. (default=None) + verify : bool + If True, uses certificate verification; if False will not do so. Only + used for 'now' tags (default=True) Raises ------ @@ -316,6 +319,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False, "%Y-%m", gfz_data_name, pds.DateOffset(months=1, seconds=-1), update_files=update_files, - mock_download_dir=mock_download_dir) + mock_download_dir=mock_download_dir, + verify=verify) return From 1b4d23d75daf02fcb8d850e2df4bb3c329cc2dd8 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 19 Oct 2026 17:57:40 -0400 Subject: [PATCH 02/24] ENH: added a verify flag Added a verify flag to the GFZ downloads. --- .../instruments/sw_methods/gfz.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/GeospaceDataManagement/instruments/sw_methods/gfz.py b/GeospaceDataManagement/instruments/sw_methods/gfz.py index 0ddcd1c..2879881 100644 --- a/GeospaceDataManagement/instruments/sw_methods/gfz.py +++ b/GeospaceDataManagement/instruments/sw_methods/gfz.py @@ -59,7 +59,7 @@ def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt, gfz_data_name, freq, update_files=False, is_def=False, - mock_download_dir=None): + mock_download_dir=None, verify=True): """Download data from GFZ into CSV files at a specified cadence. Parameters @@ -89,6 +89,9 @@ def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt, Local directory with downloaded files or None. If not None, will process any files with the correct name and date (following the local file prefix and date format) as if they were downloaded (default=None) + verify : bool + If True, uses certificate verification; if False will not do so + (default=True) Raises ------ @@ -131,7 +134,7 @@ def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt, query_url = '{:s}&status=def'.format(query_url) # The data is returned as a JSON file - req = requests.get(query_url, verify=False) + req = requests.get(query_url, verify=verify) # Process the JSON file if req.text.find('Gateway Timeout') >= 0: @@ -175,7 +178,7 @@ def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt, def kp_ap_cp_download(platform, name, date_array, tag, inst_id, data_path, - mock_download_dir=None): + mock_download_dir=None, verify=True): """Download Kp, ap, and Cp data from GFZ. Parameters @@ -197,6 +200,9 @@ def kp_ap_cp_download(platform, name, date_array, tag, inst_id, data_path, Local directory with downloaded files or None. If not None, will process any files with the correct name and date (following the local file prefix and date format) as if they were downloaded (default=None) + verify : bool + If True, uses certificate verification; if False will not do so + (default=True) Raises ------ @@ -252,7 +258,7 @@ def kp_ap_cp_download(platform, name, date_array, tag, inst_id, data_path, dl_date.strftime('%Y')))) if mock_download_dir is None: furl = ''.join([burl, fname]) - req = requests.get(furl, verify=False) + req = requests.get(furl, verify=verify) raw_txt = req.text if req.ok else None else: From 6532126e943cce40332ad9525041fbf459fc8fde Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 19 Oct 2026 17:58:07 -0400 Subject: [PATCH 03/24] BUG: check for existing metadata Check that the meta data exists before accessing it. --- GeospaceDataManagement/instruments/sw_methods/kp_ap.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GeospaceDataManagement/instruments/sw_methods/kp_ap.py b/GeospaceDataManagement/instruments/sw_methods/kp_ap.py index 33ffe85..c3c8274 100644 --- a/GeospaceDataManagement/instruments/sw_methods/kp_ap.py +++ b/GeospaceDataManagement/instruments/sw_methods/kp_ap.py @@ -270,7 +270,10 @@ def ap(kk): var_name)) # Convert from Kp to ap - fill_val = kp_inst[var_name].attrs[fill_label] + if fill_label in kp_inst[var_name].attrs: + fill_val = kp_inst[var_name].attrs[fill_label] + else: + fill_val = gdm.utils.meta.default_fill_values_from_type(float) ap_data = np.array([ap(kp) if kp != fill_val else fill_val for kp in kp_inst[var_name].values]) From e4a6d248d3c479a505c286051455597e5e627d81 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 19 Oct 2026 17:59:32 -0400 Subject: [PATCH 04/24] BUG: fix meta assignment and access Fixed bugs where meta data was not transferred when combining data, updated the default index name for F10.7 data, and removed attempted access to the pysat Meta class. --- .../instruments/sw_methods/f107.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/GeospaceDataManagement/instruments/sw_methods/f107.py b/GeospaceDataManagement/instruments/sw_methods/f107.py index b5b4a2c..721d522 100644 --- a/GeospaceDataManagement/instruments/sw_methods/f107.py +++ b/GeospaceDataManagement/instruments/sw_methods/f107.py @@ -169,6 +169,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None, f107_times = list() f107_values = list() + f107_attrs = dict() # Cycle through the desired time range itime = dt.datetime(start.year, start.month, start.day) @@ -221,6 +222,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None, good_vals]) f107_values.extend(new_vals) itime = f107_times[-1] + pds.DateOffset(days=1) + f107_attrs.update(standard_inst['f107'].attrs) else: inst_flag = 'forecast' notes += "{:})".format(itime.date()) @@ -281,6 +283,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None, forecast_inst['f107'][good_times].values[good_vals]) f107_values.extend(new_vals) itime = f107_times[-1] + pds.DateOffset(days=1) + f107_attrs.update(forecast_inst['f107'].attrs) notes += "{:})".format(itime.date()) @@ -290,7 +293,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None, notes += "{:})".format(itime.date()) if index_name is None: - index_name = 'time' + index_name = 'Epoch' if fill_val is None: fill_val = default_fill_values_from_type(float) @@ -328,7 +331,8 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None, f107_values.extend([fill_val for kk in extend_times]) # Save output data - data = xr.Dataset(data_vars={'f107': ((index_name), f107_values)}, + data = xr.Dataset(data_vars={'f107': ((index_name), f107_values, + f107_attrs)}, coords={index_name: f107_times}) # Resample the output data, filling missing values @@ -381,7 +385,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41, f107a_name)) if fill_label in f107_inst[f107_name].attrs: - fill_val = f107_inst.meta[f107_name].attrs[fill_label] + fill_val = f107_inst[f107_name].attrs[fill_label] else: fill_val = default_fill_values_from_type(float) @@ -434,7 +438,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41, # There may be missing days in the output data, remove these if f107_inst.index.shape < f107_fill[time_name].shape: - f107_fill = f107_fill.loc[f107_inst.index] + f107_fill = f107_fill.loc[{f107_inst.index.name: f107_inst.index}] # Update the metadata if len(f107_inst.index) > 1: From 4204808879ffcf6ba1670f1fc85d31de823fd008 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 19 Oct 2026 18:02:59 -0400 Subject: [PATCH 05/24] DOC: update changelog Add a summary of changes to the changelog. --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b797b52..977d8c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +[0.0.3] - 2026-10-XX +-------------------- +* ENH: + - Added the ability to turn GFZ download certificate verification on and off +* BUG: + - Fixed F10.7 data concatonation (default index name, assigning meta data) + - Ensure Kp meta data exists before attempting access + [0.0.2] - 2026-10-02 -------------------- * ENH: From 808f669b59a5ae8692c11f77de7a010876ddf66e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 21 Oct 2026 13:16:33 -0400 Subject: [PATCH 06/24] DOC: fixed grammar Fixed wording of 'verify' description. --- GeospaceDataManagement/instruments/sw_ssn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GeospaceDataManagement/instruments/sw_ssn.py b/GeospaceDataManagement/instruments/sw_ssn.py index 2af87c4..25cd1cb 100644 --- a/GeospaceDataManagement/instruments/sw_ssn.py +++ b/GeospaceDataManagement/instruments/sw_ssn.py @@ -283,7 +283,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False, downloaded. (default=None) verify : bool If True, uses certificate verification; if False will not do so. Only - used for 'now' tags (default=True) + used for the 'now' tag (default=True) Raises ------ From e4cd58504963359a7d11c5f3042d2f22b4f53c3c Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 21 Oct 2026 13:17:12 -0400 Subject: [PATCH 07/24] TST: added f107 method tests Added tests for the F10.7 index name and presence of attributes after using the combine function. --- GeospaceDataManagement/tests/test_sw_methods_f107.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/GeospaceDataManagement/tests/test_sw_methods_f107.py b/GeospaceDataManagement/tests/test_sw_methods_f107.py index 0122341..e959316 100644 --- a/GeospaceDataManagement/tests/test_sw_methods_f107.py +++ b/GeospaceDataManagement/tests/test_sw_methods_f107.py @@ -223,6 +223,7 @@ def test_combine_f107_no_data_no_files(self): # Test the output assert np.isnan(f107_inst.data["f107"]).all() + assert f107_inst.index.name == "Epoch" del combo_in, f107_inst return @@ -258,5 +259,9 @@ def test_combine_f107_all(self): assert len(f107_inst.vars_no_time) == 1 assert 'f107' in f107_inst.variables + # Test attributes + for akey in self.combine_inst['historic']['f107'].attrs.keys(): + assert akey in f107_inst['f107'].attrs + del f107_inst return From 84f42b23621754cc0f7117a122d1976df1a3a2d0 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 21 Oct 2026 13:17:41 -0400 Subject: [PATCH 08/24] TST: added kp method test Added a test for the convert Kp routine in the absence of a fill value attribute. --- .../tests/test_sw_methods_kp.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/GeospaceDataManagement/tests/test_sw_methods_kp.py b/GeospaceDataManagement/tests/test_sw_methods_kp.py index 681d76c..767e877 100644 --- a/GeospaceDataManagement/tests/test_sw_methods_kp.py +++ b/GeospaceDataManagement/tests/test_sw_methods_kp.py @@ -226,6 +226,23 @@ def test_convert_kp_to_ap(self): assert self.inst['3hr_ap'].max() <= self.inst['3hr_ap'].attrs['max_val'] return + def test_convert_kp_to_ap_no_fill_attr(self): + """Test conversion of Kp to ap with no Kp fill value attribute.""" + # Remove the fill value attribute + del self.inst['Kp'].attrs['fill_val'] + assert 'fill_val' not in self.inst['Kp'].attrs + + # Run the conversion + kp_ap.convert_3hr_kp_to_ap(self.inst) + + # Test the output, which will have a fill value attribute set to NaN + assert '3hr_ap' in self.inst.variables + assert len(self.inst['3hr_ap'].attrs) > 0 + assert self.inst['3hr_ap'].min() >= self.inst['3hr_ap'].attrs['min_val'] + assert self.inst['3hr_ap'].max() <= self.inst['3hr_ap'].attrs['max_val'] + assert np.isnan(self.inst['3hr_ap'].attrs['fill_val']) + return + def test_convert_kp_to_ap_fill_val(self): """Test conversion of Kp to ap with fill values.""" From 82ebdc56dfbb826b7b7e21bff13f1eac330b84c4 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 23 Oct 2026 11:06:25 -0400 Subject: [PATCH 09/24] ENH: added check for downloaded files Added a check and logger warning for instances where the downloaded files do not match the requested range of files. --- GeospaceDataManagement/_instrument.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/GeospaceDataManagement/_instrument.py b/GeospaceDataManagement/_instrument.py index d1a3351..f0ef11f 100644 --- a/GeospaceDataManagement/_instrument.py +++ b/GeospaceDataManagement/_instrument.py @@ -3306,6 +3306,17 @@ def download(self, start=None, stop=None, date_array=None, self.bounds = (self.files[dsel1].iloc[0], self.files[dsel2].iloc[-1], curr_bound[2], curr_bound[3]) + + # Check the updated file range to see if it covers the requested + # download period + missing_dates = [adate for adate in date_array + if adate not in self.files.files.index] + if len(missing_dates) > 0: + gdm.logger.warning(''.join([ + 'Unable to download {:d} days '.format(len(missing_dates)), + 'between ', repr(missing_dates[0]), ' and ', + repr(missing_dates[-1]), '; this may be related to the ', + 'coverage of the remote repository'])) else: gdm.logger.warning('Requested download over an empty date range.') From 2e0e78a9f9a76fd775761f4c45852cefacacba53 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 23 Oct 2026 11:07:32 -0400 Subject: [PATCH 10/24] BUG: fix file refresh for empty instrument Fixed the evaluation of files when attempting to refresh an empty instrument; allows the code to gracefully inform the user of the lack of files. --- GeospaceDataManagement/_files.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/GeospaceDataManagement/_files.py b/GeospaceDataManagement/_files.py index cc59a19..c8158aa 100644 --- a/GeospaceDataManagement/_files.py +++ b/GeospaceDataManagement/_files.py @@ -776,16 +776,19 @@ def refresh(self): # there truly a point to this? return + if new_files is None: + new_files = pds.Series() + # Ensure the name of returned Series is None for consistency new_files.name = None - # If we find some files, this is the one directory we store. - # If I don't remove the directory paths then loading by filename + # If we find some files, this is the one directory we store. If we + # don't remove the directory paths then loading by filename # becomes more of a challenge. Plus, more memory to store, more # difficult for a human to parse when browsing a list, etc. The # approach here provides for most of the potential functionality # of multiple directories while still leaving the 'single' directory - # focus and features of the original GeospaceDataManagement intact. + # focus intact. if not new_files.empty: self.data_path = path new_files = self._remove_data_dir_path(new_files) From 7b481c33ac2904fffd14cb4eae58dfb0acef5519 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 23 Oct 2026 11:07:53 -0400 Subject: [PATCH 11/24] TST: added test for new logger warning Added a test for the new download logger warning. --- GeospaceDataManagement/tests/test_instrument.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/GeospaceDataManagement/tests/test_instrument.py b/GeospaceDataManagement/tests/test_instrument.py index 26fb93c..e80a78b 100644 --- a/GeospaceDataManagement/tests/test_instrument.py +++ b/GeospaceDataManagement/tests/test_instrument.py @@ -16,6 +16,7 @@ import datetime as dt from importlib import reload +import logging import numpy as np import pandas as pds import pytest @@ -491,6 +492,18 @@ def test_load_empty_instrument_no_files_error(self): warn_type=UserWarning) return + def test_load_empty_instrument_no_downloads_warning(self, caplog): + """Test logger warning when downloading from empty repository.""" + + # Trying to download without any files to get + with caplog.at_level(logging.WARNING, logger='GeospaceDataManagement'): + self.empty_inst.download(start=self.empty_inst.today()) + + # Test the warning + captured = caplog.text + assert captured.find('be related to the coverage of the remote') > 0 + return + def test_empty_repr_eval(self): """Test that repr functions on empty `Instrument`.""" From 16ebe327eaae0cbed954562102860522fccdd07f Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 23 Oct 2026 11:08:17 -0400 Subject: [PATCH 12/24] DOC: updated changelog Added the enhancement to the changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 977d8c3..d093116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). -------------------- * ENH: - Added the ability to turn GFZ download certificate verification on and off + - Added a warning if some of the requested download days were not retrieved * BUG: - Fixed F10.7 data concatonation (default index name, assigning meta data) - Ensure Kp meta data exists before attempting access From 4f5ab772ec7c9688617f5b7c5958aa6c8601fb09 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 27 Oct 2026 18:17:41 -0400 Subject: [PATCH 13/24] DOC: update the SWPC links The SWPC webpage has moved, update the links. --- docs/instruments/sw_instruments.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/instruments/sw_instruments.rst b/docs/instruments/sw_instruments.rst index 35306e0..184fb5d 100644 --- a/docs/instruments/sw_instruments.rst +++ b/docs/instruments/sw_instruments.rst @@ -17,7 +17,7 @@ historic measurements of the solar wind. This differs from the ACE scientific data, which is available at a greater latency from `CDAWeb `_. Information about these data sets can be found at the `SWPC ACE Solar-Wind page -`_. +`_. .. _ace-epam-inst: @@ -163,7 +163,7 @@ Ap is a geomagnetic index that reflects the magnitude of geomagnetic disturbances at Earth but unlike Kp uses a linear scale. Historic, recent (last 30 days), and forecasted values are available from the `GFZ Kp `_ and the -`SWPC Forecasts page `_. +`SWPC Forecasts page `_. .. automodule:: GeospaceDataManagement.instruments.sw_ap @@ -229,7 +229,7 @@ sfu) `[Cortie 1912] `_. Historic indices, real-time indices, and forecasted indices are available from the `LASP F107 `_ and the -`SWPC F107 page `_. +`SWPC F107 page `_. .. automodule:: GeospaceDataManagement.instruments.sw_f107 @@ -244,7 +244,7 @@ Solar Flares Solar flares have been monitored for decades and the data has been compiled into standard measurements from different data sets. Historic indices, real-time indices, and forecasted indices are available from -`SWPC `_. +`SWPC `_. .. automodule:: GeospaceDataManagement.instruments.sw_flare @@ -275,8 +275,8 @@ Kp Kp is a geomagnetic index that reflects the magnitude of geomagnetic disturbances at Earth. Historic, recent (last 30 days), and forecasted values are available from the -`GFZ Kp repository `_, and the -`SWPC Kp page `_. +`GFZ Kp repository `_, and the +`SWPC Kp page `_. .. automodule:: GeospaceDataManagement.instruments.sw_kp @@ -332,7 +332,7 @@ Sunspot Number The Sunspot Number (SSN) is one of the oldest continuously measured solar indices. Currently, this Instrument provides preliminary and daily values from SWPC (for example, here is the -`forecast page `_). +`forecast page `_). .. automodule:: GeospaceDataManagement.instruments.sw_ssn :members: @@ -345,7 +345,7 @@ Storm Probability Geomagnetic storm predictions are provided by SWPC for global, high-latitude, and mid-latitude regions. SWPC uses the -`NOAA SW scales `_, which +`NOAA SW scales `_, which are explained here. .. automodule:: GeospaceDataManagement.instruments.sw_stormprob From 9b9a8d73e75f15f9e68dff4e823b6ec17a325da2 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 12:28:51 -0400 Subject: [PATCH 14/24] DOC: updated link Updated a redirecting link. --- docs/instruments/sw_instruments.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/instruments/sw_instruments.rst b/docs/instruments/sw_instruments.rst index 184fb5d..c4bd744 100644 --- a/docs/instruments/sw_instruments.rst +++ b/docs/instruments/sw_instruments.rst @@ -225,7 +225,8 @@ F \ :sub:`10.7`\ ^^^^^^^^^^^^^^^^ F \ :sub:`10.7`\ is the 10.7 cm radio solar flux (measured in solar flux units, -sfu) `[Cortie 1912] `_. +sfu) +`[Cortie 1912] `_. Historic indices, real-time indices, and forecasted indices are available from the `LASP F107 `_ and the From 12bfd673e2e877a4ae77103a7e75e456a8b12d06 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 12:29:10 -0400 Subject: [PATCH 15/24] DOC: updated a changed link Updated the redirecting link. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef831f2..91ab98d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,7 @@ If you are proposing a new feature or a change in something that already exists: To set up `GeospaceDataManagement` for local development: -1. [Fork GeospaceDataManagement on GitHub](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo). +1. [Fork GeospaceDataManagement on GitHub](https://docs.github.com/en/pull-requests/how-tos/work-with-forks/fork-a-repo). 2. Clone your fork locally: From e25fccd5b45db59a535201e0ed2bd19a8b89f932 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 12:29:44 -0400 Subject: [PATCH 16/24] DOC: updated changelog Added the documentation update to the changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d093116..d32ffd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * BUG: - Fixed F10.7 data concatonation (default index name, assigning meta data) - Ensure Kp meta data exists before attempting access +* DOC: + - Updated redirecting links [0.0.2] - 2026-10-02 -------------------- From b3ade691f4727e380143beb1784f27c83a2f0a5c Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 12:30:08 -0400 Subject: [PATCH 17/24] BUG: windows errors are different An attempted fix for the windows implementation. --- GeospaceDataManagement/_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GeospaceDataManagement/_instrument.py b/GeospaceDataManagement/_instrument.py index f0ef11f..aa9e142 100644 --- a/GeospaceDataManagement/_instrument.py +++ b/GeospaceDataManagement/_instrument.py @@ -3217,7 +3217,7 @@ def download(self, start=None, stop=None, date_array=None, # Make sure directories are there, otherwise create them try: os.makedirs(self.files.data_path) - except OSError as err: + except (OSError, FileNotFoundError) as err: if err.errno != errno.EEXIST: # Ok if directories already exist, otherwise exit with an # error that includes the message from original error. From 706c85a60780f1fb4d256a484c86da104d84fd7b Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 12:36:17 -0400 Subject: [PATCH 18/24] BUG: fix numpy deprecation warning Numpy timedelta64 now requires a unit specification. --- GeospaceDataManagement/tests/test_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GeospaceDataManagement/tests/test_utils.py b/GeospaceDataManagement/tests/test_utils.py index 183ba56..9490fba 100644 --- a/GeospaceDataManagement/tests/test_utils.py +++ b/GeospaceDataManagement/tests/test_utils.py @@ -282,14 +282,14 @@ def test_listify_failure_with_dict_iterable(self, iterable): return @pytest.mark.parametrize('iterable', [ - np.timedelta64(1), np.full((1, 1), np.timedelta64(1)), - np.full((2, 2), np.timedelta64(1)), - np.full((3, 3, 3), np.timedelta64(1))]) + np.timedelta64(1, 's'), np.full((1, 1), np.timedelta64(1, 's')), + np.full((2, 2), np.timedelta64(1, 's')), + np.full((3, 3, 3), np.timedelta64(1, 's'))]) def test_listify_class_arrays(self, iterable): """Test listify with various np.arrays of classes.""" new_iterable = utils.listify(iterable) - tst_iterable = [np.timedelta64(1) + tst_iterable = [np.timedelta64(1, 's') for i in range(int(np.prod(np.shape(iterable))))] utils.testing.assert_lists_equal(new_iterable, tst_iterable) return From 7de2df9b21f2c7a34eaa9f2aac7f2b6f6f5ff948 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 13:10:20 -0400 Subject: [PATCH 19/24] DOC: update old link Update a redirecting link. --- docs/tutorial/tutorial_basics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/tutorial_basics.rst b/docs/tutorial/tutorial_basics.rst index c264c85..434e2a2 100644 --- a/docs/tutorial/tutorial_basics.rst +++ b/docs/tutorial/tutorial_basics.rst @@ -192,7 +192,7 @@ GeospaceDataManagement methods. DMSP data is hosted by the `Madrigal database `_, a community resource for geospace data. The proper process for downloading DMSP and other Madrigal data is built into the open source tool -`madrigalWeb `_, +`madrigalWeb `_, which is invoked appropriately by GeospaceDataManagement within the :py:mod:`dmsp_ivm` sub-module. Madrigal requires that users provide their name and email address as their username and password. From a66fd654c33232b364df4161043efcda6f16f1b4 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 13:10:50 -0400 Subject: [PATCH 20/24] BUG: fix error message The Windows FileNotFoundError class doesn't have a message attribute. --- GeospaceDataManagement/_instrument.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GeospaceDataManagement/_instrument.py b/GeospaceDataManagement/_instrument.py index aa9e142..e7f5e70 100644 --- a/GeospaceDataManagement/_instrument.py +++ b/GeospaceDataManagement/_instrument.py @@ -3217,14 +3217,15 @@ def download(self, start=None, stop=None, date_array=None, # Make sure directories are there, otherwise create them try: os.makedirs(self.files.data_path) - except (OSError, FileNotFoundError) as err: + except OSError as err: if err.errno != errno.EEXIST: # Ok if directories already exist, otherwise exit with an # error that includes the message from original error. msg = ''.join(('There was a problem creating the path: ', self.files.data_path, ', to store downloaded data for ', self.platform, - self.name, '. ', err.message)) + self.name, '. ', err.message + if hasattr(err, 'message') else repr(err))) raise ValueError(msg) if start is None and stop is None and date_array is None: From a1dc7e9c5b902f7b4cb4f2f670bd36ec9b619691 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 13:57:35 -0400 Subject: [PATCH 21/24] BUG: fixed empty subdirectory path If any entry in `os.path.join` is a single directory separator, the entire path is set to be just that. Update the subdirectory path to be empty instead of a directory separator if there is no subdirectory structure. --- GeospaceDataManagement/_files.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/GeospaceDataManagement/_files.py b/GeospaceDataManagement/_files.py index c8158aa..00903c2 100644 --- a/GeospaceDataManagement/_files.py +++ b/GeospaceDataManagement/_files.py @@ -210,6 +210,11 @@ def __init__(self, inst, data_dir=None, directory_format=None, self.sub_dir_path = os.path.normpath( self.directory_format.format(**self.inst_info)) + if len(self.sub_dir_path) == 1: + # If this is just a single directory separator, os.path.join will + # die. Reset as an empty string to get the same effect + self.sub_dir_path = '' + # Ensure we have at least one path for gdm data directory if len(gdm.params['data_dirs']) == 0: raise NameError("".join(("GeospaceDataManagement's `data_dirs` ", From aefac35582cdedb6f0081366bfbdbed0e765bc39 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 14:20:58 -0400 Subject: [PATCH 22/24] BUG: only rename variables that change When removing a prefix that may not be in all variable names, do not rename variables whose names are unchanged. --- GeospaceDataManagement/instruments/methods/general.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/GeospaceDataManagement/instruments/methods/general.py b/GeospaceDataManagement/instruments/methods/general.py index cf65110..a295ee6 100644 --- a/GeospaceDataManagement/instruments/methods/general.py +++ b/GeospaceDataManagement/instruments/methods/general.py @@ -184,8 +184,12 @@ def remove_leading_text(inst, target=None): else: map_keys = {} for key in inst.data.variables.keys(): - map_keys[key] = key.split(prepend_str)[-1] - inst.data = inst.data.rename(name_dict=map_keys) + new_key = key.split(prepend_str)[-1] + if key != new_key: + map_keys[key] = new_key + + if len(map_keys.keys()) > 0: + inst.data = inst.data.rename(name_dict=map_keys) return From f1155506f6adeac4d66ea60e4f8ed811bdf9c163 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 14:21:15 -0400 Subject: [PATCH 23/24] DOC: update changelog Add additional changes to the changelog. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d32ffd3..a021f0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ This project adheres to [Semantic Versioning](https://semver.org/). * BUG: - Fixed F10.7 data concatonation (default index name, assigning meta data) - Ensure Kp meta data exists before attempting access + - Fixed bug in assigning Files attributes when they are empty + - When updating variable names in the general instrument methods, do not + attempt to rename variables whose names are the same. * DOC: - Updated redirecting links From 7ca79761dad3ac8e6f2b92bf37fd25f89b0365b2 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 28 Oct 2026 15:39:01 -0400 Subject: [PATCH 24/24] BUG: Windows bug for directory seperators Because `//var/whatever` and `/var/whatever` are different locations in Windows, `os.path.normpath` won't remove multiple directory separators. Changed the logic to look for a single unique character instead of a single character. --- GeospaceDataManagement/_files.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/GeospaceDataManagement/_files.py b/GeospaceDataManagement/_files.py index 00903c2..89cd5dc 100644 --- a/GeospaceDataManagement/_files.py +++ b/GeospaceDataManagement/_files.py @@ -210,9 +210,11 @@ def __init__(self, inst, data_dir=None, directory_format=None, self.sub_dir_path = os.path.normpath( self.directory_format.format(**self.inst_info)) - if len(self.sub_dir_path) == 1: - # If this is just a single directory separator, os.path.join will - # die. Reset as an empty string to get the same effect + if len(set(self.sub_dir_path)) == 1: + # If this is just directory separator(s), os.path.join will + # die. Reset as an empty string to get the same effect. Windows + # can have multiple directory seporators, so the logic looks for + # one unique character self.sub_dir_path = '' # Ensure we have at least one path for gdm data directory