diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index 1e675151..b95055f2 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -319,7 +319,17 @@ else if (stringValue != null) } } - if (value == null) + // Blank in the source arrives as a null-valued row; remove the value it replaces. A lookup miss (below) keeps the old value. + if (floatValue == null && dateTimeValue == null && stringValue == null) + { + if (isUpdate) + { + OntologyObject ontologyObject = OntologyManager.getOntologyObject(container, objectURI); + if (null != ontologyObject) + OntologyManager.deleteProperty(ontologyObject, OntologyManager.getPropertyDescriptor(pd.getPropertyURI(), container), false); + } + } + else if (value == null) { if (pd.getLookupSchema() != null && pd.getLookupQuery() != null) { diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index d26a3e35..bcbd72b5 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -25,7 +25,6 @@ import org.labkey.api.data.JdbcType; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.dataiterator.DataIteratorContext; @@ -49,7 +48,6 @@ import java.io.IOException; import java.sql.SQLException; -import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -111,9 +109,6 @@ public QueryUpdateService getUpdateService() protected static class UpdateService extends SNDQueryUpdateService { - /** Keeps the ObjectURI IN clause well under the SQL Server parameter limit. */ - private static final int URI_CHUNK_SIZE = 500; - private final SNDManager _sndManager = SNDManager.get(); private final SNDService _sndService = SNDService.get(); private final DbSchema _expSchema = OntologyManager.getExpSchema(); @@ -128,62 +123,6 @@ private String getObjectURI(Integer eventDataId, Container c) return _sndManager.generateLsid(c, String.valueOf(eventDataId)); } - /** - * EventDataIds in this batch whose exp.Object currently carries attribute values. Deleting the exp.Object row - * cascades to exp.ObjectProperty, so these are the values the merge destroys; only the _SND Attribute Data ETL - * step re-inserts them, and it computes its incremental window independently of this step's. - */ - private Set getEventDataIdsWithAttributeData(Container container, Map eventDataIdsByUri) - { - Set withAttributeData = new HashSet<>(); - List uris = new ArrayList<>(eventDataIdsByUri.keySet()); - - for (int i = 0; i < uris.size(); i += URI_CHUNK_SIZE) - { - List chunk = uris.subList(i, Math.min(i + URI_CHUNK_SIZE, uris.size())); - - // EXISTS rather than a join: both indexes (UQ_Object on ObjectURI, PK_ObjectProperty on ObjectId) - // are seeks, and the semi-join stops at the first property instead of reading all of them per object. - SQLFragment sql = new SQLFragment("SELECT o.ObjectURI FROM ") - .append(OntologyManager.getTinfoObject(), "o") - .append(" WHERE o.Container = ?").add(container.getId()) - .append(" AND EXISTS (SELECT 1 FROM ").append(OntologyManager.getTinfoObjectProperty(), "op") - .append(" WHERE op.ObjectId = o.ObjectId)") - .append(" AND o.ObjectURI").appendInClause(chunk, _expSchema.getSqlDialect()); - - new SqlSelector(_expSchema, sql).getCollection(String.class) - .forEach(uri -> withAttributeData.add(eventDataIdsByUri.get(uri))); - } - - return withAttributeData; - } - - /** - * Diagnostic only, so a failure here must not abort the merge. Skipped above the cap logIds lists at, where the - * chunked queries would cost hundreds of round trips to produce a bare count. - */ - private void logAttributeDataToBeCleared(Container container, Map eventDataIdsByUri, Logger log) - { - if (!log.isDebugEnabled()) - return; - - if (eventDataIdsByUri.size() > SNDManager.MAX_LOGGED_IDS) - { - log.debug("More than " + SNDManager.MAX_LOGGED_IDS + " EventDataIds in this batch; skipping the check for attribute values about to be cleared."); - return; - } - - try - { - SNDManager.logIds(log, "Attribute values about to be cleared by this merge; the _SND Attribute Data step must re-insert them.", - getEventDataIdsWithAttributeData(container, eventDataIdsByUri)); - } - catch (Exception e) - { - log.debug("Could not determine which EventDataIds have attribute values; continuing with the merge.", e); - } - } - @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) @@ -235,17 +174,14 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B SNDManager.logIds(log, "EventDataIds merged into snd.EventData by this batch:", eventDataIdsByUri.values()); SNDManager.logRowversionRange(log, "Source span of the merged rows.", data, SNDManager.PROC_ROWVERSION_COLUMN); - logAttributeDataToBeCleared(container, eventDataIdsByUri, log); int count = 0; for(Map map : data) { String objectURI = (String) map.get("ObjectURI"); - //delete row from exp.Object - OntologyManager.deleteOntologyObjects(container, objectURI); - - //add updated row to exp.Object + // Never delete the exp.Object row first: that cascades this row's attribute values away, and only the + // _SND Attribute Data step re-inserts them, on an incremental window computed independently of this one. OntologyManager.ensureObject(container, objectURI); //add to list of cached narrative rows to delete diff --git a/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql b/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql index 5886e528..fe8e07a8 100644 --- a/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql +++ b/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql @@ -37,6 +37,7 @@ AS -- 04/23/2024 Lookup values need to be string values by default. tjh -- 6/26/2025 Added check for eventId in labkey Events table. tjh -- 8/28/2026 Added ProcRowversion and AttribRowversion so the ETL step can log the rowversion it filtered on and which row it came from. +-- 9/9/2026 Blank attribute values pass through as NULL-valued rows instead of being filtered out, so the ETL removes the stored value. -- ========================================================================================== SELECT TOP (99.999999999) PERCENT cp.ANIMAL_EVENT_ID AS EventId, @@ -45,15 +46,15 @@ SELECT TOP (99.999999999) PERCENT cp.PROC_ID AS EventDataId, sp.SUPER_PKG_ID AS SuperPkgId, pbi.SUPER_PKG_ID AS ParentSuperPkgId, - cpa.value AS value, + val.v AS value, -- exp.ObjectProperty columns LTRIM(RTRIM(cpa.ATTRIB_KEY)) AS [_KEY], CASE WHEN ( (LOWER(pa.DATA_TYPE) = 'numeric' OR LOWER(pa.DATA_TYPE) = 'decimal') ) AND pa.LOOKUP_KEY IS NULL - THEN CAST (REPLACE(cpa.value,',','') AS FLOAT ) ELSE NULL END AS FloatValue, + THEN CAST (REPLACE(val.v,',','') AS FLOAT ) ELSE NULL END AS FloatValue, CASE WHEN LOWER(pa.DATA_TYPE) = 'string' OR pa.LOOKUP_KEY IS NOT NULL - THEN cpa.value ELSE NULL END AS StringValue, + THEN val.v ELSE NULL END AS StringValue, CASE WHEN ( (LOWER(pa.DATA_TYPE)) = 'string' OR pa.LOOKUP_KEY IS NOT NULL) THEN 's' ELSE 'f' END AS TypeTag, @@ -74,12 +75,13 @@ INNER JOIN dbo.BUDGET_ITEMS AS pbi ON pbi.BUDGET_ITEM_ID = bi.PARENT_BUDGET_ITEM INNER JOIN dbo.SUPER_PKGS AS sp ON sp.SUPER_PKG_ID = bi.SUPER_PKG_ID INNER JOIN dbo.PKGS AS p ON p.PKG_ID = sp.PKG_ID INNER JOIN dbo.PKG_ATTRIBS AS pa ON pa.PKG_ID = p.PKG_ID AND pa.ATTRIB_KEY = cpa.ATTRIB_KEY +-- A blank is NULL here rather than a filtered-out row, so the ETL learns the attribute was cleared. CAST('' AS FLOAT) would otherwise read as 0. +CROSS APPLY (SELECT CASE WHEN LTRIM(RTRIM(cpa.VALUE)) = '' THEN NULL ELSE cpa.VALUE END AS v) AS val -- select primates only from the TxBiomed colony INNER JOIN labkey_etl.V_DEMOGRAPHICS AS D ON D.id = ae.ANIMAL_ID -WHERE LTRIM(RTRIM(cpa.VALUE)) <> '' AND cpa.VALUE IS NOT NULL -- limit selection to only events that have been imported -AND EXISTS (SELECT 1 FROM labkey.snd.Events AS e WHERE cp.ANIMAL_EVENT_ID = e.EventId) +WHERE EXISTS (SELECT 1 FROM labkey.snd.Events AS e WHERE cp.ANIMAL_EVENT_ID = e.EventId) ORDER BY EventDataId GO