From dabc04ea37089e8ac98d124d40877a3526047e34 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Fri, 21 Aug 2026 12:11:08 +0100 Subject: [PATCH 1/6] Allow to store null to include sync-only SubmitHosts in site sync refresh --- monitoring/db_update_sqlite.py | 12 +++++++++--- monitoring/publishing/models.py | 4 ++-- .../publishing/templates/gridsync_submithost.html | 4 ++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 7548c2c4..54c17ab7 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -494,11 +494,17 @@ def refresh_gridsitesync_submithost(): year = record.get("Year") submit_host = record.get("SubmitHostSumm") or record.get("SubmitHostSync") record_start = record.get("RecordStart") + record_end = record.get("RecordEnd") - # Skip rows where RecordStart is missing or NaN - if pd.isna(record_start): + if pd.isna(submit_host): continue + if pd.isna(record_start): + record_start = None + + if pd.isna(record_end): + record_end = None + # Sanitize numeric fields record_count_published = record.get("RecordCountPublished") if pd.isna(record_count_published): @@ -516,7 +522,7 @@ def refresh_gridsitesync_submithost(): SubmitHost=submit_host, defaults={ 'RecordStart': record_start, - 'RecordEnd': record.get("RecordEnd"), + 'RecordEnd': record_end, 'RecordCountPublished': record_count_published, 'RecordCountInDb': record_count_in_db, } diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index 195872db..0007f1ce 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -86,8 +86,8 @@ class GridSiteSyncSubmitH(models.Model): YearMonth = models.CharField(max_length=255) Year = models.IntegerField() Month = models.IntegerField() - RecordStart = models.DateTimeField() - RecordEnd = models.DateTimeField() + RecordStart = models.DateTimeField(null=True, blank=True) + RecordEnd = models.DateTimeField(null=True, blank=True) RecordCountPublished = models.IntegerField() RecordCountInDb = models.IntegerField() SubmitHost = models.CharField(max_length=255) diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html index 0baf9aaa..d43d0f33 100644 --- a/monitoring/publishing/templates/gridsync_submithost.html +++ b/monitoring/publishing/templates/gridsync_submithost.html @@ -35,8 +35,8 @@

APEL Synchronisation Test

{{ host.YearMonth }} {{ host.SubmitHost }} - {{ host.RecordStart }} - {{ host.RecordEnd }} + {{ host.RecordStart|default:"n/a" }} + {{ host.RecordEnd|default:"n/a" }} {{ host.RecordCountInDb|intcomma }} {{ host.RecordCountPublished|intcomma }} From 16f868794c0bbb4d42e066b3aeb0c113a325558e Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Mon, 24 Aug 2026 09:10:29 +0100 Subject: [PATCH 2/6] Remove white space to fix pre-commit failure --- monitoring/db_update_sqlite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 54c17ab7..719eeaad 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -501,9 +501,9 @@ def refresh_gridsitesync_submithost(): if pd.isna(record_start): record_start = None - + if pd.isna(record_end): - record_end = None + record_end = None # Sanitize numeric fields record_count_published = record.get("RecordCountPublished") From aa0f1d6694d3f2c0887794dd8ab0190eeeb126c4 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Mon, 24 Aug 2026 10:18:55 +0100 Subject: [PATCH 3/6] Use SubmitHostSync when SubmitHostSumm is missing (NaN) --- monitoring/db_update_sqlite.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 719eeaad..cf0b9289 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -492,7 +492,12 @@ def refresh_gridsitesync_submithost(): site = record.get("Site") month = record.get("Month") year = record.get("Year") - submit_host = record.get("SubmitHostSumm") or record.get("SubmitHostSync") + # Use SubmitHostSync when SubmitHostSumm is missing (NaN) + submit_host = ( + record.get("SubmitHostSync") + if pd.isna(record.get("SubmitHostSumm")) + else record.get("SubmitHostSumm") + ) record_start = record.get("RecordStart") record_end = record.get("RecordEnd") @@ -503,7 +508,7 @@ def refresh_gridsitesync_submithost(): record_start = None if pd.isna(record_end): - record_end = None + record_end = None # Sanitize numeric fields record_count_published = record.get("RecordCountPublished") From 555b319f50cbdf3f04507c7bc139621f51c220a3 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Wed, 26 Aug 2026 10:35:07 +0100 Subject: [PATCH 4/6] Remove blank=True as not needed --- monitoring/publishing/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index 0007f1ce..27de0579 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -86,8 +86,8 @@ class GridSiteSyncSubmitH(models.Model): YearMonth = models.CharField(max_length=255) Year = models.IntegerField() Month = models.IntegerField() - RecordStart = models.DateTimeField(null=True, blank=True) - RecordEnd = models.DateTimeField(null=True, blank=True) + RecordStart = models.DateTimeField(null=True) + RecordEnd = models.DateTimeField(null=True) RecordCountPublished = models.IntegerField() RecordCountInDb = models.IntegerField() SubmitHost = models.CharField(max_length=255) From 81f27831a7b9014f2abef2f5d5ff10887f04d10a Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Wed, 26 Aug 2026 10:36:46 +0100 Subject: [PATCH 5/6] Check null at the initial fetch using a helper function to make code readable --- monitoring/db_update_sqlite.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index cf0b9289..c206a088 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -498,18 +498,12 @@ def refresh_gridsitesync_submithost(): if pd.isna(record.get("SubmitHostSumm")) else record.get("SubmitHostSumm") ) - record_start = record.get("RecordStart") - record_end = record.get("RecordEnd") + record_start = none_if_missing(record.get("RecordStart")) + record_end = none_if_missing(record.get("RecordEnd")) if pd.isna(submit_host): continue - if pd.isna(record_start): - record_start = None - - if pd.isna(record_end): - record_end = None - # Sanitize numeric fields record_count_published = record.get("RecordCountPublished") if pd.isna(record_count_published): @@ -539,6 +533,10 @@ def refresh_gridsitesync_submithost(): log.exception("Error while trying to refresh GridSiteSyncSubmitH") +def none_if_missing(value): + return None if pd.isna(value) else value + + if __name__ == "__main__": log.info('=====================') From 3a5b6a018f2e1313a537577b22945b14fe43fa2c Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 27 Aug 2026 09:29:45 +0100 Subject: [PATCH 6/6] Document helper purpose to improve readability and maintenance. --- monitoring/db_update_sqlite.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index c206a088..bf90547f 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -534,6 +534,9 @@ def refresh_gridsitesync_submithost(): def none_if_missing(value): + """ + Return None when the value is missing (NaN/NaT), otherwise return it unchanged. + """ return None if pd.isna(value) else value