Space Weather Bugs

CHANGELOG.md

@@ -3,6 +3,20 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/).
 
+[0.0.3] - 2026-10-XX
+--------------------
+* ENH:
+  - Added the ability to turn GFZ download certificate verification on and off
+  - Added a warning if some of the requested download days were not retrieved
+* BUG:
+  - Fixed F10.7 data concatonation (default index name, assigning meta data)
+  - Ensure Kp meta data exists before attempting access
+  - Fixed bug in assigning Files attributes when they are empty
+  - When updating variable names in the general instrument methods, do not
+    attempt to rename variables whose names are the same.
+* DOC:
+  - Updated redirecting links
+
 [0.0.2] - 2026-10-02
 --------------------
 * ENH:

CONTRIBUTING.md

@@ -47,7 +47,7 @@ If you are proposing a new feature or a change in something that already exists:
 
 To set up `GeospaceDataManagement` for local development:
 
-1. [Fork GeospaceDataManagement on GitHub](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo).
+1. [Fork GeospaceDataManagement on GitHub](https://docs.github.com/en/pull-requests/how-tos/work-with-forks/fork-a-repo).
 
 2. Clone your fork locally:
 

GeospaceDataManagement/_files.py

@@ -210,6 +210,13 @@ def __init__(self, inst, data_dir=None, directory_format=None,
         self.sub_dir_path = os.path.normpath(
             self.directory_format.format(**self.inst_info))
 
+        if len(set(self.sub_dir_path)) == 1:
+            # If this is just directory separator(s), os.path.join will
+            # die. Reset as an empty string to get the same effect. Windows
+            # can have multiple directory seporators, so the logic looks for
+            # one unique character
+            self.sub_dir_path = ''
+
         # Ensure we have at least one path for gdm data directory
         if len(gdm.params['data_dirs']) == 0:
             raise NameError("".join(("GeospaceDataManagement's `data_dirs` ",
@@ -776,16 +783,19 @@ def refresh(self):
                 # there truly a point to this?
                 return
 
+            if new_files is None:
+                new_files = pds.Series()
+
             # Ensure the name of returned Series is None for consistency
             new_files.name = None
 
-            # If we find some files, this is the one directory we store.
-            # If I don't remove the directory paths then loading by filename
+            # If we find some files, this is the one directory we store. If we
+            # don't remove the directory paths then loading by filename
             # becomes more of a challenge. Plus, more memory to store, more
             # difficult for a human to parse when browsing a list, etc. The
             # approach here provides for most of the potential functionality
             # of multiple directories while still leaving the 'single' directory
-            # focus and features of the original GeospaceDataManagement intact.
+            # focus intact.
             if not new_files.empty:
                 self.data_path = path
                 new_files = self._remove_data_dir_path(new_files)

GeospaceDataManagement/_instrument.py

@@ -3224,7 +3224,8 @@ def download(self, start=None, stop=None, date_array=None,
                 msg = ''.join(('There was a problem creating the path: ',
                                self.files.data_path,
                                ', to store downloaded data for ', self.platform,
-                               self.name, '. ', err.message))
+                               self.name, '. ', err.message
+                               if hasattr(err, 'message') else repr(err)))
                 raise ValueError(msg)
 
         if start is None and stop is None and date_array is None:
@@ -3306,6 +3307,17 @@ def download(self, start=None, stop=None, date_array=None,
                         self.bounds = (self.files[dsel1].iloc[0],
                                        self.files[dsel2].iloc[-1],
                                        curr_bound[2], curr_bound[3])
+
+            # Check the updated file range to see if it covers the requested
+            # download period
+            missing_dates = [adate for adate in date_array
+                             if adate not in self.files.files.index]
+            if len(missing_dates) > 0:
+                gdm.logger.warning(''.join([
+                    'Unable to download {:d} days '.format(len(missing_dates)),
+                    'between ', repr(missing_dates[0]), ' and ',
+                    repr(missing_dates[-1]), '; this may be related to the ',
+                    'coverage of the remote repository']))
         else:
             gdm.logger.warning('Requested download over an empty date range.')
 

GeospaceDataManagement/instruments/methods/general.py

@@ -184,8 +184,12 @@ def remove_leading_text(inst, target=None):
         else:
             map_keys = {}
             for key in inst.data.variables.keys():
-                map_keys[key] = key.split(prepend_str)[-1]
-            inst.data = inst.data.rename(name_dict=map_keys)
+                new_key = key.split(prepend_str)[-1]
+                if key != new_key:
+                    map_keys[key] = new_key
+
+            if len(map_keys.keys()) > 0:
+                inst.data = inst.data.rename(name_dict=map_keys)
 
     return
 

GeospaceDataManagement/instruments/sw_ap.py

@@ -227,7 +227,8 @@ def list_files(tag='', inst_id='', data_path='', format_str=None):
     return files
 
 
-def download(date_array, tag, inst_id, data_path, mock_download_dir=None):
+def download(date_array, tag, inst_id, data_path, mock_download_dir=None,
+             verify=True):
     """Download the Ap index data from the appropriate repository.
 
     Parameters
@@ -244,6 +245,9 @@ def download(date_array, tag, inst_id, data_path, mock_download_dir=None):
         Local directory with downloaded files or None. If not None, will
         process any files with the correct name and date as if they were
         downloaded (default=None)
+    verify : bool
+        If True, uses certificate verification; if False will not do so. Only
+        used for 'def' and 'now' tags (default=True)
 
     Note
     ----
@@ -262,7 +266,8 @@ def download(date_array, tag, inst_id, data_path, mock_download_dir=None):
 
     if tag in ['def', 'now']:
         sw_methods.gfz.kp_ap_cp_download(platform, name, date_array, tag,
-                                         inst_id, data_path, mock_download_dir)
+                                         inst_id, data_path, mock_download_dir,
+                                         verify=verify)
     elif tag == 'recent':
         sw_methods.swpc.kp_ap_recent_download(name, date_array, data_path,
                                               mock_download_dir)

GeospaceDataManagement/instruments/sw_apo.py

@@ -170,7 +170,7 @@ def list_files(tag='', inst_id='', data_path='', format_str=None):
 
 
 def download(date_array, tag, inst_id, data_path, update_files=False,
-             mock_download_dir=None):
+             mock_download_dir=None, verify=True):
     """Download the apo index data from the appropriate repository.
 
     Parameters
@@ -189,6 +189,9 @@ def download(date_array, tag, inst_id, data_path, update_files=False,
         Local directory with downloaded files or None. If not None, will
         process any files with the correct name and date as if they were
         downloaded (default=None)
+    verify : bool
+        If True, uses certificate verification; if False will not do so.
+        (default=True)
 
     Note
     ----
@@ -214,6 +217,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False,
     sw_methods.gfz.json_downloads(date_array, data_path, local_file_prefix,
                                   local_date_fmt, gfz_data_name, freq,
                                   update_files=update_files,
-                                  mock_download_dir=mock_download_dir)
+                                  mock_download_dir=mock_download_dir,
+                                  verify=verify)
 
     return

GeospaceDataManagement/instruments/sw_f107.py

@@ -341,7 +341,7 @@ def list_files(tag='', inst_id='', data_path='', format_str=None):
 
 
 def download(date_array, tag, inst_id, data_path, update_files=False,
-             mock_download_dir=None):
+             mock_download_dir=None, verify=True):
     """Download F107 index data from the appropriate repository.
 
     Parameters
@@ -360,6 +360,9 @@ def download(date_array, tag, inst_id, data_path, update_files=False,
         Local directory with downloaded files or None. If not None, will
         process any files with the correct name and date as if they were
         downloaded (default=None)
+    verify : bool
+        If True, uses certificate verification; if False will not do so. Only
+        used for the 'now' tag (default=True)
 
     Raises
     ------
@@ -415,7 +418,8 @@ def download(date_array, tag, inst_id, data_path, update_files=False,
                                       "%Y-%m", gfz_data_name,
                                       pds.DateOffset(months=1, seconds=-1),
                                       update_files=update_files,
-                                      mock_download_dir=mock_download_dir)
+                                      mock_download_dir=mock_download_dir,
+                                      verify=verify)
 
     elif tag == 'daily':
         sw_methods.swpc.daily_dsd_download(name, today, data_path,

GeospaceDataManagement/instruments/sw_hpo.py

@@ -170,7 +170,7 @@ def list_files(tag='', inst_id='', data_path='', format_str=None):
 
 
 def download(date_array, tag, inst_id, data_path, update_files=False,
-             mock_download_dir=None):
+             mock_download_dir=None, verify=True):
     """Download the Hpo index data from the appropriate repository.
 
     Parameters
@@ -189,6 +189,9 @@ def download(date_array, tag, inst_id, data_path, update_files=False,
         Local directory with downloaded files or None. If not None, will
         process any files with the correct name and date as if they were
         downloaded (default=None)
+    verify : bool
+        If True, uses certificate verification; if False will not do so
+        (default=True)
 
     Note
     ----
@@ -215,6 +218,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False,
     sw_methods.gfz.json_downloads(date_array, data_path, local_file_prefix,
                                   local_date_fmt, gfz_data_name, freq,
                                   update_files=update_files,
-                                  mock_download_dir=mock_download_dir)
+                                  mock_download_dir=mock_download_dir,
+                                  verify=verify)
 
     return

GeospaceDataManagement/instruments/sw_kp.py

@@ -218,7 +218,8 @@ def list_files(tag='', inst_id='', data_path='', format_str=None):
     return files
 
 
-def download(date_array, tag, inst_id, data_path, mock_download_dir=None):
+def download(date_array, tag, inst_id, data_path, mock_download_dir=None,
+             verify=True):
     """Download the Kp index data from the appropriate repository.
 
     Parameters
@@ -235,6 +236,9 @@ def download(date_array, tag, inst_id, data_path, mock_download_dir=None):
         Local directory with downloaded files or None. If not None, will
         process any files with the correct name and date as if they were
         downloaded (default=None)
+    verify : bool
+        If True, uses certificate verification; if False will not do so. Only
+        used for 'def' and 'now' tags (default=True)
 
     Raises
     ------
@@ -255,7 +259,8 @@ def download(date_array, tag, inst_id, data_path, mock_download_dir=None):
     # Download standard Kp data
     if tag in ['def', 'now']:
         sw_methods.gfz.kp_ap_cp_download(platform, name, date_array, tag,
-                                         inst_id, data_path, mock_download_dir)
+                                         inst_id, data_path, mock_download_dir,
+                                         verify=verify)
     elif tag == 'forecast':
         sw_methods.swpc.geomag_forecast_download(name, date_array, data_path,
                                                  mock_download_dir)

GeospaceDataManagement/instruments/sw_methods/f107.py

@@ -169,6 +169,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None,
 
     f107_times = list()
     f107_values = list()
+    f107_attrs = dict()
 
     # Cycle through the desired time range
     itime = dt.datetime(start.year, start.month, start.day)
@@ -221,6 +222,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None,
                     good_vals])
                 f107_values.extend(new_vals)
                 itime = f107_times[-1] + pds.DateOffset(days=1)
+                f107_attrs.update(standard_inst['f107'].attrs)
             else:
                 inst_flag = 'forecast'
                 notes += "{:})".format(itime.date())
@@ -281,6 +283,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None,
                         forecast_inst['f107'][good_times].values[good_vals])
                     f107_values.extend(new_vals)
                     itime = f107_times[-1] + pds.DateOffset(days=1)
+                    f107_attrs.update(forecast_inst['f107'].attrs)
 
             notes += "{:})".format(itime.date())
 
@@ -290,7 +293,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None,
         notes += "{:})".format(itime.date())
 
     if index_name is None:
-        index_name = 'time'
+        index_name = 'Epoch'
     if fill_val is None:
         fill_val = default_fill_values_from_type(float)
 
@@ -328,7 +331,8 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None,
         f107_values.extend([fill_val for kk in extend_times])
 
     # Save output data
-    data = xr.Dataset(data_vars={'f107': ((index_name), f107_values)},
+    data = xr.Dataset(data_vars={'f107': ((index_name), f107_values,
+                                          f107_attrs)},
                       coords={index_name: f107_times})
 
     # Resample the output data, filling missing values
@@ -381,7 +385,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41,
             f107a_name))
 
     if fill_label in f107_inst[f107_name].attrs:
-        fill_val = f107_inst.meta[f107_name].attrs[fill_label]
+        fill_val = f107_inst[f107_name].attrs[fill_label]
     else:
         fill_val = default_fill_values_from_type(float)
 
@@ -434,7 +438,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41,
 
     # There may be missing days in the output data, remove these
     if f107_inst.index.shape < f107_fill[time_name].shape:
-        f107_fill = f107_fill.loc[f107_inst.index]
+        f107_fill = f107_fill.loc[{f107_inst.index.name: f107_inst.index}]
 
     # Update the metadata
     if len(f107_inst.index) > 1:

GeospaceDataManagement/instruments/sw_methods/gfz.py

@@ -59,7 +59,7 @@
 
 def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt,
                    gfz_data_name, freq, update_files=False, is_def=False,
-                   mock_download_dir=None):
+                   mock_download_dir=None, verify=True):
     """Download data from GFZ into CSV files at a specified cadence.
 
     Parameters
@@ -89,6 +89,9 @@ def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt,
         Local directory with downloaded files or None. If not None, will
         process any files with the correct name and date (following the local
         file prefix and date format) as if they were downloaded (default=None)
+    verify : bool
+        If True, uses certificate verification; if False will not do so
+        (default=True)
 
     Raises
     ------
@@ -131,7 +134,7 @@ def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt,
                     query_url = '{:s}&status=def'.format(query_url)
 
                 # The data is returned as a JSON file
-                req = requests.get(query_url, verify=False)
+                req = requests.get(query_url, verify=verify)
 
                 # Process the JSON file
                 if req.text.find('Gateway Timeout') >= 0:
@@ -175,7 +178,7 @@ def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt,
 
 
 def kp_ap_cp_download(platform, name, date_array, tag, inst_id, data_path,
-                      mock_download_dir=None):
+                      mock_download_dir=None, verify=True):
     """Download Kp, ap, and Cp data from GFZ.
 
     Parameters
@@ -197,6 +200,9 @@ def kp_ap_cp_download(platform, name, date_array, tag, inst_id, data_path,
         Local directory with downloaded files or None. If not None, will
         process any files with the correct name and date (following the local
         file prefix and date format) as if they were downloaded (default=None)
+    verify : bool
+        If True, uses certificate verification; if False will not do so
+        (default=True)
 
     Raises
     ------
@@ -252,7 +258,7 @@ def kp_ap_cp_download(platform, name, date_array, tag, inst_id, data_path,
                                       dl_date.strftime('%Y'))))
             if mock_download_dir is None:
                 furl = ''.join([burl, fname])
-                req = requests.get(furl, verify=False)
+                req = requests.get(furl, verify=verify)
 
                 raw_txt = req.text if req.ok else None
             else:

GeospaceDataManagement/instruments/sw_methods/kp_ap.py

@@ -270,7 +270,10 @@ def ap(kk):
             var_name))
 
     # Convert from Kp to ap
-    fill_val = kp_inst[var_name].attrs[fill_label]
+    if fill_label in kp_inst[var_name].attrs:
+        fill_val = kp_inst[var_name].attrs[fill_label]
+    else:
+        fill_val = gdm.utils.meta.default_fill_values_from_type(float)
     ap_data = np.array([ap(kp) if kp != fill_val else fill_val
                         for kp in kp_inst[var_name].values])