From 4535fd20a06d865b47b086ad12649b044ab8b2b2 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Fri, 4 Sep 2026 04:53:06 -0600 Subject: [PATCH 1/2] Stop the SND event data merge clearing attribute values The merge deleted and recreated each row's exp.Object, cascading its attribute values away for only the _SND Attribute Data step to re-insert; because that step computes its incremental window independently, it can skip rows this merge just cleared and the values are then lost permanently. Also drops the diagnostic that reported which rows were about to be cleared, which no longer describes anything. --- .../org/labkey/snd/query/EventDataTable.java | 68 +------------------ 1 file changed, 2 insertions(+), 66 deletions(-) diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index d26a3e35..bcbd72b5 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -25,7 +25,6 @@ import org.labkey.api.data.JdbcType; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.dataiterator.DataIteratorContext; @@ -49,7 +48,6 @@ import java.io.IOException; import java.sql.SQLException; -import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -111,9 +109,6 @@ public QueryUpdateService getUpdateService() protected static class UpdateService extends SNDQueryUpdateService { - /** Keeps the ObjectURI IN clause well under the SQL Server parameter limit. */ - private static final int URI_CHUNK_SIZE = 500; - private final SNDManager _sndManager = SNDManager.get(); private final SNDService _sndService = SNDService.get(); private final DbSchema _expSchema = OntologyManager.getExpSchema(); @@ -128,62 +123,6 @@ private String getObjectURI(Integer eventDataId, Container c) return _sndManager.generateLsid(c, String.valueOf(eventDataId)); } - /** - * EventDataIds in this batch whose exp.Object currently carries attribute values. Deleting the exp.Object row - * cascades to exp.ObjectProperty, so these are the values the merge destroys; only the _SND Attribute Data ETL - * step re-inserts them, and it computes its incremental window independently of this step's. - */ - private Set getEventDataIdsWithAttributeData(Container container, Map eventDataIdsByUri) - { - Set withAttributeData = new HashSet<>(); - List uris = new ArrayList<>(eventDataIdsByUri.keySet()); - - for (int i = 0; i < uris.size(); i += URI_CHUNK_SIZE) - { - List chunk = uris.subList(i, Math.min(i + URI_CHUNK_SIZE, uris.size())); - - // EXISTS rather than a join: both indexes (UQ_Object on ObjectURI, PK_ObjectProperty on ObjectId) - // are seeks, and the semi-join stops at the first property instead of reading all of them per object. - SQLFragment sql = new SQLFragment("SELECT o.ObjectURI FROM ") - .append(OntologyManager.getTinfoObject(), "o") - .append(" WHERE o.Container = ?").add(container.getId()) - .append(" AND EXISTS (SELECT 1 FROM ").append(OntologyManager.getTinfoObjectProperty(), "op") - .append(" WHERE op.ObjectId = o.ObjectId)") - .append(" AND o.ObjectURI").appendInClause(chunk, _expSchema.getSqlDialect()); - - new SqlSelector(_expSchema, sql).getCollection(String.class) - .forEach(uri -> withAttributeData.add(eventDataIdsByUri.get(uri))); - } - - return withAttributeData; - } - - /** - * Diagnostic only, so a failure here must not abort the merge. Skipped above the cap logIds lists at, where the - * chunked queries would cost hundreds of round trips to produce a bare count. - */ - private void logAttributeDataToBeCleared(Container container, Map eventDataIdsByUri, Logger log) - { - if (!log.isDebugEnabled()) - return; - - if (eventDataIdsByUri.size() > SNDManager.MAX_LOGGED_IDS) - { - log.debug("More than " + SNDManager.MAX_LOGGED_IDS + " EventDataIds in this batch; skipping the check for attribute values about to be cleared."); - return; - } - - try - { - SNDManager.logIds(log, "Attribute values about to be cleared by this merge; the _SND Attribute Data step must re-insert them.", - getEventDataIdsWithAttributeData(container, eventDataIdsByUri)); - } - catch (Exception e) - { - log.debug("Could not determine which EventDataIds have attribute values; continuing with the merge.", e); - } - } - @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) @@ -235,17 +174,14 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B SNDManager.logIds(log, "EventDataIds merged into snd.EventData by this batch:", eventDataIdsByUri.values()); SNDManager.logRowversionRange(log, "Source span of the merged rows.", data, SNDManager.PROC_ROWVERSION_COLUMN); - logAttributeDataToBeCleared(container, eventDataIdsByUri, log); int count = 0; for(Map map : data) { String objectURI = (String) map.get("ObjectURI"); - //delete row from exp.Object - OntologyManager.deleteOntologyObjects(container, objectURI); - - //add updated row to exp.Object + // Never delete the exp.Object row first: that cascades this row's attribute values away, and only the + // _SND Attribute Data step re-inserts them, on an incremental window computed independently of this one. OntologyManager.ensureObject(container, objectURI); //add to list of cached narrative rows to delete From 06dac3f30d3f2fba52733066780cd04c182254dc Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Wed, 9 Sep 2026 06:07:27 -0600 Subject: [PATCH 2/2] Remove a stored attribute value when the source blanks it v_snd_attributeData filtered blank values out, so a cleared attribute never reached the ETL and its old value survived; the view now passes those rows through with NULL values and the attribute step deletes the property they replace. Lookup misses still keep the old value. --- snd/src/org/labkey/snd/query/AttributeDataTable.java | 12 +++++++++++- .../source_queries/create_v_snd_attributeData.sql | 12 +++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index 1e675151..b95055f2 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -319,7 +319,17 @@ else if (stringValue != null) } } - if (value == null) + // Blank in the source arrives as a null-valued row; remove the value it replaces. A lookup miss (below) keeps the old value. + if (floatValue == null && dateTimeValue == null && stringValue == null) + { + if (isUpdate) + { + OntologyObject ontologyObject = OntologyManager.getOntologyObject(container, objectURI); + if (null != ontologyObject) + OntologyManager.deleteProperty(ontologyObject, OntologyManager.getPropertyDescriptor(pd.getPropertyURI(), container), false); + } + } + else if (value == null) { if (pd.getLookupSchema() != null && pd.getLookupQuery() != null) { diff --git a/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql b/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql index 5886e528..fe8e07a8 100644 --- a/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql +++ b/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql @@ -37,6 +37,7 @@ AS -- 04/23/2024 Lookup values need to be string values by default. tjh -- 6/26/2025 Added check for eventId in labkey Events table. tjh -- 8/28/2026 Added ProcRowversion and AttribRowversion so the ETL step can log the rowversion it filtered on and which row it came from. +-- 9/9/2026 Blank attribute values pass through as NULL-valued rows instead of being filtered out, so the ETL removes the stored value. -- ========================================================================================== SELECT TOP (99.999999999) PERCENT cp.ANIMAL_EVENT_ID AS EventId, @@ -45,15 +46,15 @@ SELECT TOP (99.999999999) PERCENT cp.PROC_ID AS EventDataId, sp.SUPER_PKG_ID AS SuperPkgId, pbi.SUPER_PKG_ID AS ParentSuperPkgId, - cpa.value AS value, + val.v AS value, -- exp.ObjectProperty columns LTRIM(RTRIM(cpa.ATTRIB_KEY)) AS [_KEY], CASE WHEN ( (LOWER(pa.DATA_TYPE) = 'numeric' OR LOWER(pa.DATA_TYPE) = 'decimal') ) AND pa.LOOKUP_KEY IS NULL - THEN CAST (REPLACE(cpa.value,',','') AS FLOAT ) ELSE NULL END AS FloatValue, + THEN CAST (REPLACE(val.v,',','') AS FLOAT ) ELSE NULL END AS FloatValue, CASE WHEN LOWER(pa.DATA_TYPE) = 'string' OR pa.LOOKUP_KEY IS NOT NULL - THEN cpa.value ELSE NULL END AS StringValue, + THEN val.v ELSE NULL END AS StringValue, CASE WHEN ( (LOWER(pa.DATA_TYPE)) = 'string' OR pa.LOOKUP_KEY IS NOT NULL) THEN 's' ELSE 'f' END AS TypeTag, @@ -74,12 +75,13 @@ INNER JOIN dbo.BUDGET_ITEMS AS pbi ON pbi.BUDGET_ITEM_ID = bi.PARENT_BUDGET_ITEM INNER JOIN dbo.SUPER_PKGS AS sp ON sp.SUPER_PKG_ID = bi.SUPER_PKG_ID INNER JOIN dbo.PKGS AS p ON p.PKG_ID = sp.PKG_ID INNER JOIN dbo.PKG_ATTRIBS AS pa ON pa.PKG_ID = p.PKG_ID AND pa.ATTRIB_KEY = cpa.ATTRIB_KEY +-- A blank is NULL here rather than a filtered-out row, so the ETL learns the attribute was cleared. CAST('' AS FLOAT) would otherwise read as 0. +CROSS APPLY (SELECT CASE WHEN LTRIM(RTRIM(cpa.VALUE)) = '' THEN NULL ELSE cpa.VALUE END AS v) AS val -- select primates only from the TxBiomed colony INNER JOIN labkey_etl.V_DEMOGRAPHICS AS D ON D.id = ae.ANIMAL_ID -WHERE LTRIM(RTRIM(cpa.VALUE)) <> '' AND cpa.VALUE IS NOT NULL -- limit selection to only events that have been imported -AND EXISTS (SELECT 1 FROM labkey.snd.Events AS e WHERE cp.ANIMAL_EVENT_ID = e.EventId) +WHERE EXISTS (SELECT 1 FROM labkey.snd.Events AS e WHERE cp.ANIMAL_EVENT_ID = e.EventId) ORDER BY EventDataId GO