From 8c5c597c90030b41da77571f0f8d26dbece45928 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 5 Nov 2025 13:47:14 -0800 Subject: [PATCH 01/10] Expand flow dataset --- SivStudies/resources/queries/study/flow.query.xml | 13 +++++++++++-- SivStudies/resources/queries/study/flow/.qview.xml | 7 +++++-- .../study/datasets/datasets_metadata.xml | 11 ++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/SivStudies/resources/queries/study/flow.query.xml b/SivStudies/resources/queries/study/flow.query.xml index 63a7946a..3ca630fe 100644 --- a/SivStudies/resources/queries/study/flow.query.xml +++ b/SivStudies/resources/queries/study/flow.query.xml @@ -9,6 +9,9 @@ Start Date Date + + Tissue + Sample Type @@ -18,8 +21,14 @@ Population - - Result + + Parent Population + + + % of Parent + + + Absolute Count Units diff --git a/SivStudies/resources/queries/study/flow/.qview.xml b/SivStudies/resources/queries/study/flow/.qview.xml index 7c9abe0b..1413e211 100644 --- a/SivStudies/resources/queries/study/flow/.qview.xml +++ b/SivStudies/resources/queries/study/flow/.qview.xml @@ -2,10 +2,13 @@ - + + - + + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index 802e195f..bd0907f2 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -509,13 +509,22 @@ varchar + + varchar + varchar varchar - + + varchar + + + double + + double From bef9938ef300b9a373b33d1a8d8d827d39d658e5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 6 Nov 2025 05:59:02 -0800 Subject: [PATCH 02/10] Change pattern used in AutoCreateDemographicsTrigger --- .../query/AutoCreateDemographicsTrigger.java | 43 +++++-------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java index 00e5b739..3d9fb725 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java +++ b/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; public class AutoCreateDemographicsTrigger extends DefaultDatasetTrigger { @@ -46,55 +47,33 @@ public Factory() } } - private static final String CACHE_KEY = "~~AutoCreateDemographicsTrigger.IdsToCreate~~"; + private final Set _idsToCheck = new CaseInsensitiveHashSet(); @Override protected void afterUpsert(TableInfo table, Container c, User user, @Nullable Map newRow, @Nullable Map oldRow, ValidationException errors, Map extraContext) throws ValidationException { - if (extraContext == null) + String idField = getIdField(c); + String id = newRow.get(idField) != null ? newRow.get(idField).toString() : null; + if (id != null) { - _log.error("extraContext is null in AutoCreateDemographicsTrigger.afterUpsert()"); - return; - } - - if (!extraContext.containsKey(AutoCreateDemographicsTrigger.CACHE_KEY)) - { - extraContext.put(CACHE_KEY, new CaseInsensitiveHashSet()); - } - - if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s) - { - String idField = getIdField(c); - String id = newRow.get(idField) != null ? newRow.get(idField).toString() : null; - if (id != null) - { - s.add(id); - } + _idsToCheck.add(id); } } @Override public void complete(TableInfo table, Container c, User user, TableInfo.TriggerType event, BatchValidationException errors, Map extraContext) { - if (extraContext == null) - { - _log.error("extraContext is null in AutoCreateDemographicsTrigger.complete()"); - return; - } - - if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s) + if (!_idsToCheck.isEmpty()) { - s = new CaseInsensitiveHashSet(s); - String idField = getIdField(c); TableInfo ti = QueryService.get().getUserSchema(user, getTargetContainer(c), "study").getTable("demographics"); - List existingIds = new TableSelector(ti, PageFlowUtil.set(idField), new SimpleFilter(FieldKey.fromString(idField), s, CompareType.IN), null).getArrayList(String.class); + List existingIds = new TableSelector(ti, PageFlowUtil.set(idField), new SimpleFilter(FieldKey.fromString(idField), _idsToCheck, CompareType.IN), null).getArrayList(String.class); - s.removeAll(existingIds); + _idsToCheck.removeAll(existingIds); - if (!s.isEmpty()) + if (!_idField.isEmpty()) { - List> toInsert = s.stream().map(id -> Map.of(idField, (Object)id)).toList(); + List> toInsert = _idsToCheck.stream().map(id -> Map.of(idField, (Object)id)).toList(); try { ti.getUpdateService().insertRows(user, c, toInsert, null, null, null); From da21f4ba8546625d860126ae3cab1647abd0695d Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 13 Nov 2025 06:21:38 -0800 Subject: [PATCH 03/10] Improve MCC query --- .../queries/study/demographicsLittermates.sql | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/mcc/resources/queries/study/demographicsLittermates.sql b/mcc/resources/queries/study/demographicsLittermates.sql index 0b3f411d..d2edb072 100644 --- a/mcc/resources/queries/study/demographicsLittermates.sql +++ b/mcc/resources/queries/study/demographicsLittermates.sql @@ -1,16 +1,10 @@ SELECT -d1.Id, -d1.litterId, + d1.Id, + d1.litterId, -GROUP_CONCAT(distinct d2.Id, ',') as litterMates + (SELECT GROUP_CONCAT(distinct d2.Id, ',') as litterMates FROM study.Demographics d2 WHERE d2.qcstate.publicdata = true AND d1.litterId = d2.litterId AND d1.id != d2.id) as litterMates FROM study.Demographics d1 -JOIN study.Demographics d2 ON (d1.litterId = d2.litterId AND d1.id != d2.id) - -WHERE - d1.qcstate.publicdata = true AND - d2.qcstate.publicdata = true - -GROUP BY d1.Id, d1.litterId \ No newline at end of file +WHERE d1.qcstate.publicdata = true From 98ec9815c8a394ee35e923ebf89eda1e72c87295 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 20 Nov 2025 13:20:50 -0800 Subject: [PATCH 04/10] Expand ETL/presentation of SIV data --- SivStudies/resources/etls/idr-data.xml | 9 +- .../study/demographics/Expanded.qview.xml | 3 +- .../demographicsChallengeAndArt.query.xml | 7 + .../study/demographicsChallengeAndArt.sql | 5 +- .../study/demographicsImmunizations.query.xml | 4 + .../study/demographicsImmunizations.sql | 2 +- .../study/demographicsInterventions.query.xml | 30 ++ .../study/demographicsInterventions.sql | 20 ++ .../queries/study/immunizations.query.xml | 12 + .../queries/study/immunizations/.qview.xml | 4 + .../study/datasets/datasets_metadata.xml | 13 +- .../etl/PerformManualIdrStepsTask.java | 256 ++++++++++++++++++ .../query/SivStudiesCustomizer.java | 28 +- 13 files changed, 384 insertions(+), 9 deletions(-) create mode 100644 SivStudies/resources/queries/study/demographicsInterventions.query.xml create mode 100644 SivStudies/resources/queries/study/demographicsInterventions.sql create mode 100644 SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml index 909ee84e..f686476f 100644 --- a/SivStudies/resources/etls/idr-data.xml +++ b/SivStudies/resources/etls/idr-data.xml @@ -100,8 +100,8 @@ - - + + @@ -155,6 +155,11 @@ + + + + + diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml index 1598334f..779d2b71 100644 --- a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml @@ -10,7 +10,8 @@ - + + diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml index 135d018b..9dbeddfa 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml @@ -10,13 +10,20 @@ SIV Infection + /query/executeQuery.view?schemaName=study&query.queryName=treatments&query.Id~eq=${Id}&query.category~eq=SIV Infection + _blank ART + /query/executeQuery.view?schemaName=study&query.queryName=treatments&query.Id~eq=${Id}&query.category~eq=ART + _blank Infection Date + + ART Initiation (WPI) + allInfections diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index 4e4f9fc7..c3896da0 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -12,6 +12,9 @@ SELECT WHEN t.category = 'SIV Infection' THEN t.date ELSE NULL END) as infectionDate, - + min(CASE + WHEN t.category = 'ART' THEN t.timePostSivChallenge.daysPostInfection + ELSE NULL + END) as artInitiationDate FROM study.treatments t GROUP BY t.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsImmunizations.query.xml b/SivStudies/resources/queries/study/demographicsImmunizations.query.xml index 20e1fe98..bd3587c5 100644 --- a/SivStudies/resources/queries/study/demographicsImmunizations.query.xml +++ b/SivStudies/resources/queries/study/demographicsImmunizations.query.xml @@ -10,9 +10,13 @@ Immunizations + /query/executeQuery.view?schemaName=study&query.queryName=immunizations&query.Id~eq=${Id} + _blank Immunization Types + /query/executeQuery.view?schemaName=study&query.queryName=immunizations&query.Id~eq=${Id} + _blank immunizations diff --git a/SivStudies/resources/queries/study/demographicsImmunizations.sql b/SivStudies/resources/queries/study/demographicsImmunizations.sql index a31a58ce..4f28d89b 100644 --- a/SivStudies/resources/queries/study/demographicsImmunizations.sql +++ b/SivStudies/resources/queries/study/demographicsImmunizations.sql @@ -1,6 +1,6 @@ SELECT s.Id, - group_concat(DISTINCT s.treatment, char(10)) as immunizations, + group_concat(DISTINCT COALESCE(s.backbone, s.treatment), char(10)) as immunizations, group_concat(DISTINCT s.category, char(10)) as immunizationTypes, FROM study.immunizations s diff --git a/SivStudies/resources/queries/study/demographicsInterventions.query.xml b/SivStudies/resources/queries/study/demographicsInterventions.query.xml new file mode 100644 index 00000000..c41a1cc7 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsInterventions.query.xml @@ -0,0 +1,30 @@ + + + + + Interventions/Treatments Summary + + + true + true + + + Interventions/Treatments + /query/executeQuery.view?schemaName=study&query.queryName=treatments&query.Id~eq=${Id}&query.category~eq=Intervention + _blank + + + Date of First Intervention + + + First Intervention (DPI) + + + First Intervention (WPI) + + + allInterventions +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsInterventions.sql b/SivStudies/resources/queries/study/demographicsInterventions.sql new file mode 100644 index 00000000..306ac17a --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsInterventions.sql @@ -0,0 +1,20 @@ +SELECT + t.Id, + group_concat(DISTINCT CASE + WHEN t.category = 'Intervention' THEN (t.treatment || ' (' || t.timePostSivChallenge.timePostInfection || ')') + ELSE NULL + END, char(10)) as allInterventions, + min(CASE + WHEN t.category = 'Intervention' THEN t.date + ELSE NULL + END) as firstInterventionDate, + min(CASE + WHEN t.category = 'Intervention' THEN t.timePostSivChallenge.daysPostInfection + ELSE NULL + END) as firstInterventionDPI, + min(CASE + WHEN t.category = 'Intervention' THEN t.timePostSivChallenge.weeksPostInfection + ELSE NULL + END) as firstInterventionWPI +FROM study.treatments t +GROUP BY t.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml index 7e1a7a60..2fc7f169 100644 --- a/SivStudies/resources/queries/study/immunizations.query.xml +++ b/SivStudies/resources/queries/study/immunizations.query.xml @@ -15,6 +15,12 @@ Treatment + + Vector Backbone + + + Antigens + Route @@ -27,6 +33,12 @@ Reason + + Vector ID + + + Is Mock? + Comments textarea diff --git a/SivStudies/resources/queries/study/immunizations/.qview.xml b/SivStudies/resources/queries/study/immunizations/.qview.xml index 575150b0..bc9ac8dc 100644 --- a/SivStudies/resources/queries/study/immunizations/.qview.xml +++ b/SivStudies/resources/queries/study/immunizations/.qview.xml @@ -7,6 +7,10 @@ + + + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index bd0907f2..2aaf20ea 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -110,13 +110,18 @@ entityid true - varchar + + varchar + varchar + + varchar + varchar @@ -129,6 +134,12 @@ varchar + + boolean + + + integer + varchar diff --git a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java new file mode 100644 index 00000000..24c8ff3b --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java @@ -0,0 +1,256 @@ +package org.labkey.sivstudies.etl; + +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.writer.ContainerUser; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PerformManualIdrStepsTask implements TaskRefTask +{ + protected ContainerUser _containerUser; + + @Override + public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJobException + { + pruneSivChallenges(pipelineJob); + updateVaccineInformation(pipelineJob); + updateChallengeAnchorDates(pipelineJob); + + return new RecordedActionSet(); + } + + private void pruneSivChallenges(PipelineJob pipelineJob) throws PipelineJobException + { + TableInfo ti = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("treatments"); + + Map> existingRecords = new HashMap<>(); + new TableSelector(ti, PageFlowUtil.set("Id", "date", "lsid"), new SimpleFilter(FieldKey.fromString("treatment"), "SIV - Unknown", CompareType.NEQ_OR_NULL).addCondition(FieldKey.fromString("category"), "SIV Infection"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("Id")); + if (!existingRecords.containsKey(id)) + { + existingRecords.put(id, new HashSet<>()); + } + + existingRecords.get(id).add(rs.getDate(FieldKey.fromString("date"))); + }); + + final List> toDelete = new ArrayList<>(); + new TableSelector(ti, PageFlowUtil.set("Id", "date", "lsid"), new SimpleFilter(FieldKey.fromString("treatment"), "SIV - Unknown"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("Id")); + if (!existingRecords.containsKey(id)) + { + return; + } + + if (existingRecords.get(id).contains(rs.getDate(FieldKey.fromString("date")))) + { + toDelete.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")))); + } + }); + + if (!toDelete.isEmpty()) + { + pipelineJob.getLogger().info("Deleting " + toDelete.size() + " SIV challenge records"); + + try + { + ti.getUpdateService().deleteRows(_containerUser.getUser(), _containerUser.getContainer(), toDelete, null, null); + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | InvalidKeyException e) + { + throw new PipelineJobException(e); + } + } + } + + private void updateVaccineInformation(PipelineJob pipelineJob) throws PipelineJobException + { + TableInfo ti = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("immunizations"); + + final List> toUpdate = new ArrayList<>(); + new TableSelector(ti, PageFlowUtil.set("lsid", "treatment", "backbone", "antigens")).forEachResults(rs -> { + String treatment = rs.getString(FieldKey.fromString("treatment")); + if (treatment == null) + { + return; + } + + String backbone = rs.getString(FieldKey.fromString("backbone")); + Map updatedRow = new HashMap<>(); + + if (backbone != null && backbone.contains("68-1")) + { + if (treatment.contains("miR-142-126")) + { + updatedRow.put("backbone", "68-1 MHC-1A-only"); + } + else if (treatment.contains("miR-126")) + { + updatedRow.put("backbone", "68-1 MHC-E-only"); + } + else if (treatment.contains("miR-142")) + { + updatedRow.put("backbone", "68-1 MHC-II-only"); + } + } + + if (treatment.toUpperCase().contains("MOCK")) + { + updatedRow.put("isMock", true); + } + + if (!updatedRow.isEmpty()) + { + updatedRow.put("lsid", rs.getString(FieldKey.fromString("lsid"))); + toUpdate.add(updatedRow); + } + }); + + if (!toUpdate.isEmpty()) + { + pipelineJob.getLogger().info("Updating " + toUpdate.size() + " immunization records"); + + try + { + BatchValidationException bve = new BatchValidationException(); + + List> oldKeys = toUpdate.stream().map(x -> Map.of("lsid", x.get("lsid"))).toList(); + ti.getUpdateService().updateRows(_containerUser.getUser(), _containerUser.getContainer(), toUpdate, oldKeys, bve, null, null); + + if (bve.hasErrors()) + { + throw bve; + } + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | InvalidKeyException e) + { + throw new PipelineJobException(e); + } + } + } + + private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws PipelineJobException + { + TableInfo treatments = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("treatments"); + TableInfo ad = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "studies").getTable("subjectAnchorDates"); + + Map> existingRecords = new HashMap<>(); + new TableSelector(ad, PageFlowUtil.set("Id", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), "SIV Infection"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("Id")); + if (!existingRecords.containsKey(id)) + { + existingRecords.put(id, new HashSet<>()); + } + + existingRecords.get(id).add(rs.getDate(FieldKey.fromString("date"))); + }); + + final Map> sourceRecords = new HashMap<>(); + final List> toInsert = new ArrayList<>(); + new TableSelector(treatments, PageFlowUtil.set("Id", "date", "objectId"), new SimpleFilter(FieldKey.fromString("category"), "SIV Infection"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("Id")); + Date date = rs.getDate(FieldKey.fromString("date")); + + if (!sourceRecords.containsKey(id)) + { + sourceRecords.put(id, new HashSet<>()); + } + sourceRecords.get(id).add(date); + + if (!existingRecords.containsKey(id) | !existingRecords.get(id).contains(date)) + { + toInsert.add(Map.of( + "Id", id, + "date", date, + "category", "SIV Infection", + "sourceRecord", rs.getString(FieldKey.fromString("objectId")) + )); + } + }); + + if (!toInsert.isEmpty()) + { + pipelineJob.getLogger().info("Inserting " + toInsert.size() + " SIV challenge anchor date records"); + + try + { + BatchValidationException bve = new BatchValidationException(); + ad.getUpdateService().insertRows(_containerUser.getUser(), _containerUser.getContainer(), toInsert, bve, null, null); + + if (bve.hasErrors()) + { + throw bve; + } + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e) + { + throw new PipelineJobException(e); + } + } + + final List> toDelete = new ArrayList<>(); + new TableSelector(ad, PageFlowUtil.set("Id", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), "SIV Infection"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("Id")); + Date date = rs.getDate(FieldKey.fromString("date")); + if (!sourceRecords.containsKey(id) | !sourceRecords.get(id).contains(date)) + { + toDelete.add(Map.of("rowId", rs.getInt(FieldKey.fromString("rowId")))); + } + }); + + if (!toDelete.isEmpty()) + { + pipelineJob.getLogger().info("Deleting " + toDelete.size() + " SIV challenge anchor date records"); + + try + { + ad.getUpdateService().deleteRows(_containerUser.getUser(), _containerUser.getContainer(), toDelete, null, null); + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | InvalidKeyException e) + { + throw new PipelineJobException(e); + } + } + } + + @Override + public List getRequiredSettings() + { + return List.of(); + } + + @Override + public void setSettings(Map map) throws XmlException + { + + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 1fc15ddb..3ba90f45 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -13,6 +13,7 @@ import org.labkey.api.data.WrappedColumn; import org.labkey.api.ldk.table.AbstractTableCustomizer; import org.labkey.api.query.ExprColumn; +import org.labkey.api.query.FieldKey; import org.labkey.api.query.LookupForeignKey; import org.labkey.api.query.QueryDefinition; import org.labkey.api.query.QueryException; @@ -27,6 +28,7 @@ import org.labkey.api.util.logging.LogHelper; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -251,6 +253,13 @@ private void appendDemographicsColumns(AbstractTableInfo parentTable) parentTable.addColumn(colInfo); } + if (parentTable.getColumn("interventions") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(parentTable.getUserSchema(), "demographicsInterventions", parentTable, "interventions"); + colInfo.setLabel("Interventions"); + parentTable.addColumn(colInfo); + } + if (parentTable.getColumn("outcomes") == null) { BaseColumnInfo colInfo = getWrappedIdCol(parentTable.getUserSchema(), "demographicsOutcomes", parentTable, "outcomes"); @@ -348,17 +357,25 @@ public TableInfo getLookupTableInfo() String name = queryName + "_sivChallenge"; UserSchema targetSchema = targetTable.getUserSchema().getDefaultSchema().getUserSchema(targetSchemaName); QueryDefinition qd = QueryService.get().createQueryDef(u, targetSchemaContainer, targetSchema, name); - qd.setSql("SELECT\n" + + qd.setSql("SELECT t.*,\n" + + "CASE\n" + + "WHEN t.daysPostInfection IS NULL THEN NULL\n" + + "WHEN t.daysPostInfection <= 28 THEN (CAST(t.daysPostInfection AS VARCHAR) || ' DPI')\n" + + "ELSE (CAST(t.weeksPostInfection AS VARCHAR) || ' WPI')\n" + + "END as timePostInfection\n" + + "FROM (SELECT\n" + "max(ad.date) as infectionDate,\n" + // NOTE: CAST() is used to ensure whole numbers "CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), INTEGER) as daysPostInfection,\n" + - "CONVERT(age_in_months(CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), FLOAT) as monthsPostInfection,\n" + + "CONVERT((CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), INTEGER) / 7), INTEGER) as weeksPostInfection,\n" + + "ROUND(CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), DOUBLE) / 7.0, 1) as weeksPostInfectionDecimal,\n" + + "CONVERT(age_in_months(CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), DOUBLE) as monthsPostInfection,\n" + "c." + pkCol.getFieldKey().toString() + "\n" + "FROM \"" + schemaName + "\".\"" + queryName + "\" c " + "JOIN studies.subjectAnchorDates ad ON (ad.subjectId = c." + idCol.getFieldKey().toSQLString() + ")\n" + "WHERE ad.eventLabel = 'SIV Infection'\n" + "GROUP BY c.date, c." + pkCol.getFieldKey().toString() + "\n" + - "HAVING count(*) = 1" + "HAVING count(*) = 1) t" ); qd.setIsTemporary(true); @@ -380,7 +397,12 @@ public TableInfo getLookupTableInfo() ((BaseColumnInfo)ti.getColumn("infectionDate")).setLabel("Infection Date"); ((BaseColumnInfo)ti.getColumn("daysPostInfection")).setLabel("Days Post-Infection"); + ((BaseColumnInfo)ti.getColumn("weeksPostInfection")).setLabel("Weeks Post-Infection"); ((BaseColumnInfo)ti.getColumn("monthsPostInfection")).setLabel("Months Post-Infection"); + + BaseColumnInfo tpi = ((BaseColumnInfo)ti.getColumn("timePostInfection")); + tpi.setLabel("Time Post-Infection"); + tpi.setSortFieldKeys(Arrays.asList(FieldKey.fromString("daysPostInfection"))); } return ti; From b3200819691d49c28b527eb7c877011c734e93a1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 20 Nov 2025 14:48:23 -0800 Subject: [PATCH 05/10] Expand ETL/presentation of SIV data --- .../study/demographicsChallengeAndArt.query.xml | 2 +- .../sivstudies/etl/PerformManualIdrStepsTask.java | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml index 9dbeddfa..afff74f3 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml @@ -22,7 +22,7 @@ Infection Date - ART Initiation (WPI) + ART Initiation (DPI) allInfections diff --git a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java index 24c8ff3b..4606f679 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java @@ -2,6 +2,7 @@ import org.apache.xmlbeans.XmlException; import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; @@ -158,8 +159,8 @@ private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws Pipeline TableInfo ad = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "studies").getTable("subjectAnchorDates"); Map> existingRecords = new HashMap<>(); - new TableSelector(ad, PageFlowUtil.set("Id", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), "SIV Infection"), null).forEachResults(rs -> { - String id = rs.getString(FieldKey.fromString("Id")); + new TableSelector(ad, PageFlowUtil.set("subjectId", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), "SIV Infection"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("subjectId")); if (!existingRecords.containsKey(id)) { existingRecords.put(id, new HashSet<>()); @@ -183,7 +184,7 @@ private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws Pipeline if (!existingRecords.containsKey(id) | !existingRecords.get(id).contains(date)) { toInsert.add(Map.of( - "Id", id, + "subjectId", id, "date", date, "category", "SIV Infection", "sourceRecord", rs.getString(FieldKey.fromString("objectId")) @@ -212,12 +213,12 @@ private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws Pipeline } final List> toDelete = new ArrayList<>(); - new TableSelector(ad, PageFlowUtil.set("Id", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), "SIV Infection"), null).forEachResults(rs -> { - String id = rs.getString(FieldKey.fromString("Id")); + new TableSelector(ad, PageFlowUtil.set("subjectId", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), "SIV Infection"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("subjectId")); Date date = rs.getDate(FieldKey.fromString("date")); if (!sourceRecords.containsKey(id) | !sourceRecords.get(id).contains(date)) { - toDelete.add(Map.of("rowId", rs.getInt(FieldKey.fromString("rowId")))); + toDelete.add(new CaseInsensitiveHashMap<>(Map.of("rowid", rs.getInt(FieldKey.fromString("rowId"))))); } }); From 2f7f15bc4078f18e323bbfdbcb3291b0849cc4ff Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 21 Nov 2025 06:42:43 -0800 Subject: [PATCH 06/10] Expand ETL/presentation of SIV data --- .../study/demographics/Expanded.qview.xml | 3 +- .../demographics/Project Summary.qview.xml | 2 - .../demographics/SIV and ART Info.qview.xml | 22 +++++++ .../demographicsChallengeAndArt.query.xml | 12 ++++ .../study/demographicsChallengeAndArt.sql | 51 ++++++++++----- .../queries/study/demographicsPVL.query.xml | 65 +++++++++++++++++++ .../queries/study/demographicsPVL.sql | 32 +++++++++ .../study/demographicsProjects.query.xml | 6 -- .../queries/study/demographicsProjects.sql | 5 +- .../etl/PerformManualIdrStepsTask.java | 38 +++++++---- .../query/SivStudiesCustomizer.java | 7 ++ 11 files changed, 202 insertions(+), 41 deletions(-) create mode 100644 SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml create mode 100644 SivStudies/resources/queries/study/demographicsPVL.query.xml create mode 100644 SivStudies/resources/queries/study/demographicsPVL.sql diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml index 779d2b71..b1db0a01 100644 --- a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml @@ -6,11 +6,12 @@ + - + diff --git a/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml b/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml index 30c79c60..8f20b4a7 100644 --- a/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml @@ -9,8 +9,6 @@ - - diff --git a/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml b/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml new file mode 100644 index 00000000..43f68638 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml @@ -0,0 +1,22 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml index afff74f3..db95fd32 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml @@ -20,10 +20,22 @@ Infection Date + Date + ART Initiation + Date + + ART Initiation (DPI) + + ART Release + Date + + + ART Release (WPI) + allInfections diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index c3896da0..e9ed1f28 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -1,20 +1,35 @@ SELECT - t.Id, - group_concat(DISTINCT CASE - WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') - ELSE NULL - END, char(10)) as allInfections, - group_concat(DISTINCT CASE - WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') - ELSE NULL - END, char(10)) as allART, - min(CASE - WHEN t.category = 'SIV Infection' THEN t.date - ELSE NULL - END) as infectionDate, - min(CASE - WHEN t.category = 'ART' THEN t.timePostSivChallenge.daysPostInfection + t.*, + TIMESTAMPDIFF('SQL_TSI_WEEK', t.infectionDate, t.artReleaseDate) as artReleaseWPI +FROM ( + SELECT + t.Id, + group_concat(DISTINCT CASE + WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') ELSE NULL - END) as artInitiationDate -FROM study.treatments t -GROUP BY t.Id \ No newline at end of file + END, char(10)) as allInfections, + min(floor(age(t.DataSets.Demographics.birth, CASE WHEN t.category = 'SIV Infection' THEN t.date ELSE NULL END))) AS ageAtInfection, + + group_concat(DISTINCT CASE + WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + ELSE NULL + END, char(10)) as allART, + min(CASE + WHEN t.category = 'SIV Infection' THEN t.date + ELSE NULL + END) as infectionDate, + min(CASE + WHEN t.category = 'ART' THEN t.date + ELSE NULL + END) as artInitiationDate, + min(CASE + WHEN t.category = 'ART' THEN t.timePostSivChallenge.daysPostInfection + ELSE NULL + END) as artInitiationDPI, + min(CASE + WHEN t.category = 'ART' THEN t.artInformation.artRelease + ELSE NULL + END) as artReleaseDate + FROM study.treatments t + GROUP BY t.Id +) t \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsPVL.query.xml b/SivStudies/resources/queries/study/demographicsPVL.query.xml new file mode 100644 index 00000000..b097b00a --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsPVL.query.xml @@ -0,0 +1,65 @@ + + + + + PVL Summary + + + true + true + + + SIV Challenge + Date + + + # PVLs + /query/executeQuery.view?schemaName=study&query.queryName=viralLoads&query.Id~eq=${Id}&query.sampleType~eq=Plasma&query.target~eq=SIV + _blank + + + First PVL + + + Last PVL + + + DPI of First PVL + + + DPI of Last PVL + + + WPI of First PVL + + + WPI of Last PVL + + + ART Release + Date + + + + # PVLs Post-ART Release + /query/executeQuery.view?schemaName=study&query.queryName=viralLoads&query.Id~eq=${Id}&query.sampleType~eq=Plasma&query.target~eq=SIV&query.date~dategte=${artRelease} + _blank + + + First PVL Post-ART Release (Weeks) + + + Last PVL Post-ART Release (Weeks) + + + First PVL Post-ART Release (Months) + + + Last PVL Post-ART Release (Months) + + + numPVL +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsPVL.sql b/SivStudies/resources/queries/study/demographicsPVL.sql new file mode 100644 index 00000000..e83aa354 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsPVL.sql @@ -0,0 +1,32 @@ +SELECT + t.Id, + count(*) AS numPVL, + min(sivChallenge) as sivChallenge, + min(t.date) as dateOfFirstPvl, + max(t.date) as dateOfLastPvl, + + min(CONVERT(CASE WHEN t.sivChallenge IS NULL THEN NULL WHEN t.date <= t.sivChallenge THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_DAY', t.sivChallenge, t.date) END, INTEGER)) as firstPvlDPI, + max(CONVERT(CASE WHEN t.sivChallenge IS NULL THEN NULL WHEN t.date <= t.sivChallenge THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_DAY', t.sivChallenge, t.date) END, INTEGER)) as lastPvlDPI, + + min(CONVERT(CASE WHEN t.sivChallenge IS NULL THEN NULL WHEN t.date <= t.sivChallenge THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_WEEK', t.sivChallenge, t.date) END, INTEGER)) as firstPvlWPI, + max(CONVERT(CASE WHEN t.sivChallenge IS NULL THEN NULL WHEN t.date <= t.sivChallenge THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_WEEK', t.sivChallenge, t.date) END, INTEGER)) as lastPvlWPI, + + min(artRelease) as artRelease, + sum(CASE WHEN (t.artRelease IS NOT NULL AND t.date > t.artRelease) THEN 1 ELSE 0 END) as numPVLPostArtRelease, + + min(CONVERT(CASE WHEN t.artRelease IS NULL THEN NULL WHEN t.date <= t.artRelease THEN NULL ELSE age_in_months(t.artRelease, t.date) END, FLOAT)) as firstPvlPostArtReleaseMonths, + max(CONVERT(CASE WHEN t.artRelease IS NULL THEN NULL WHEN t.date <= t.artRelease THEN NULL ELSE age_in_months(t.artRelease, t.date) END, FLOAT)) as lastPvlPostArtReleaseMonths, + + min(CONVERT(CASE WHEN t.artRelease IS NULL THEN NULL WHEN t.date <= t.artRelease THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_WEEK', t.artRelease, t.date) END, INTEGER)) as firstPvlPostArtReleaseWeeks, + max(CONVERT(CASE WHEN t.artRelease IS NULL THEN NULL WHEN t.date <= t.artRelease THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_WEEK', t.artRelease, t.date) END, INTEGER)) as lastPvlPostArtReleaseWeeks + +FROM (SELECT + vl.Id, + vl.date, + (SELECT min(tr.date) as sivChallenge FROM study.treatments tr WHERE tr.category = 'SIV Infection' AND tr.Id = vl.Id) as sivChallenge, + (SELECT max(tr.enddate) as artRelease FROM study.treatments tr WHERE tr.category = 'ART' AND tr.Id = vl.Id) as artRelease + +FROM study.viralLoads vl +WHERE vl.target = 'SIV' AND vl.sampleType = 'Plasma' +) t +GROUP BY t.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsProjects.query.xml b/SivStudies/resources/queries/study/demographicsProjects.query.xml index ec3dddc0..40389751 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.query.xml +++ b/SivStudies/resources/queries/study/demographicsProjects.query.xml @@ -17,12 +17,6 @@ Subgroups/Treatments - - RhCMV Vaccines? - - - SIV/ART Projects? - categories diff --git a/SivStudies/resources/queries/study/demographicsProjects.sql b/SivStudies/resources/queries/study/demographicsProjects.sql index 853db7d1..d1a8689e 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.sql +++ b/SivStudies/resources/queries/study/demographicsProjects.sql @@ -2,10 +2,9 @@ SELECT s.Id, count(s.Id) as totalProjects, group_concat(DISTINCT s.study, char(10)) as allStudies, + group_concat(DISTINCT s.cohortId.studyId.description, char(10)) as studyDescription, group_concat(DISTINCT s.category, char(10)) as categories, - group_concat(DISTINCT s.subgroup, char(10)) as subgroups, + group_concat(DISTINCT s.subgroup, char(10)) as subgroups - GROUP_CONCAT(distinct CASE WHEN s.category = 'RhCMV-Vaccines' THEN 'Yes' ELSE null END, char(10)) as rhCmvVaccines, - GROUP_CONCAT(distinct CASE WHEN s.category = 'SIV/ART' THEN 'Yes' ELSE null END, char(10)) as sivArt FROM study.assignment s GROUP BY s.Id \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java index 4606f679..abb8aae2 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java @@ -39,6 +39,7 @@ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJo pruneSivChallenges(pipelineJob); updateVaccineInformation(pipelineJob); updateChallengeAnchorDates(pipelineJob); + updateArtInitiationAnchorDates(pipelineJob); return new RecordedActionSet(); } @@ -68,7 +69,7 @@ private void pruneSivChallenges(PipelineJob pipelineJob) throws PipelineJobExcep if (existingRecords.get(id).contains(rs.getDate(FieldKey.fromString("date")))) { - toDelete.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")))); + toDelete.add(new CaseInsensitiveHashMap<>(Map.of("lsid", rs.getString(FieldKey.fromString("lsid"))))); } }); @@ -138,7 +139,7 @@ else if (treatment.contains("miR-142")) { BatchValidationException bve = new BatchValidationException(); - List> oldKeys = toUpdate.stream().map(x -> Map.of("lsid", x.get("lsid"))).toList(); + List> oldKeys = toUpdate.stream().map(x -> (Map)new CaseInsensitiveHashMap<>(Map.of("lsid", x.get("lsid")))).toList(); ti.getUpdateService().updateRows(_containerUser.getUser(), _containerUser.getContainer(), toUpdate, oldKeys, bve, null, null); if (bve.hasErrors()) @@ -154,12 +155,23 @@ else if (treatment.contains("miR-142")) } private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws PipelineJobException + { + updateAnchorDates(pipelineJob, "SIV Infection", "SIV Infection", "date"); + } + + private void updateArtInitiationAnchorDates(PipelineJob pipelineJob) throws PipelineJobException + { + updateAnchorDates(pipelineJob, "ART Initiation", "ART", "date"); + updateAnchorDates(pipelineJob, "ART End", "ART", "enddate"); + } + + private void updateAnchorDates(PipelineJob pipelineJob, String eventType, String treatmentCategory, String sourceDateField) throws PipelineJobException { TableInfo treatments = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("treatments"); TableInfo ad = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "studies").getTable("subjectAnchorDates"); Map> existingRecords = new HashMap<>(); - new TableSelector(ad, PageFlowUtil.set("subjectId", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), "SIV Infection"), null).forEachResults(rs -> { + new TableSelector(ad, PageFlowUtil.set("subjectId", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), eventType), null).forEachResults(rs -> { String id = rs.getString(FieldKey.fromString("subjectId")); if (!existingRecords.containsKey(id)) { @@ -171,9 +183,13 @@ private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws Pipeline final Map> sourceRecords = new HashMap<>(); final List> toInsert = new ArrayList<>(); - new TableSelector(treatments, PageFlowUtil.set("Id", "date", "objectId"), new SimpleFilter(FieldKey.fromString("category"), "SIV Infection"), null).forEachResults(rs -> { + new TableSelector(treatments, PageFlowUtil.set("Id", "date", "objectId"), new SimpleFilter(FieldKey.fromString("category"), treatmentCategory), null).forEachResults(rs -> { String id = rs.getString(FieldKey.fromString("Id")); - Date date = rs.getDate(FieldKey.fromString("date")); + Date date = rs.getDate(FieldKey.fromString(sourceDateField)); + if (date == null) + { + return; + } if (!sourceRecords.containsKey(id)) { @@ -183,18 +199,18 @@ private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws Pipeline if (!existingRecords.containsKey(id) | !existingRecords.get(id).contains(date)) { - toInsert.add(Map.of( + toInsert.add(new CaseInsensitiveHashMap<>(Map.of( "subjectId", id, "date", date, - "category", "SIV Infection", + "category", eventType, "sourceRecord", rs.getString(FieldKey.fromString("objectId")) - )); + ))); } }); if (!toInsert.isEmpty()) { - pipelineJob.getLogger().info("Inserting " + toInsert.size() + " SIV challenge anchor date records"); + pipelineJob.getLogger().info("Inserting " + toInsert.size() + " " + eventType + " anchor date records"); try { @@ -213,7 +229,7 @@ private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws Pipeline } final List> toDelete = new ArrayList<>(); - new TableSelector(ad, PageFlowUtil.set("subjectId", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), "SIV Infection"), null).forEachResults(rs -> { + new TableSelector(ad, PageFlowUtil.set("subjectId", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), eventType), null).forEachResults(rs -> { String id = rs.getString(FieldKey.fromString("subjectId")); Date date = rs.getDate(FieldKey.fromString("date")); if (!sourceRecords.containsKey(id) | !sourceRecords.get(id).contains(date)) @@ -224,7 +240,7 @@ private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws Pipeline if (!toDelete.isEmpty()) { - pipelineJob.getLogger().info("Deleting " + toDelete.size() + " SIV challenge anchor date records"); + pipelineJob.getLogger().info("Deleting " + toDelete.size() + " " + eventType + " anchor date records"); try { diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 3ba90f45..8237763c 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -273,6 +273,13 @@ private void appendDemographicsColumns(AbstractTableInfo parentTable) colInfo.setLabel("SIV/ART Dates"); parentTable.addColumn(colInfo); } + + if (parentTable.getColumn("pvlInfo") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(parentTable.getUserSchema(), "demographicsPVL", parentTable, "pvlInfo"); + colInfo.setLabel("PVL Info"); + parentTable.addColumn(colInfo); + } } private void appendPvlColumns(DatasetTable ds, String subjectColName, String dateColName) From 90d9f0444c0acc747a41b2c6a86291ad93ec4031 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 21 Nov 2025 07:47:14 -0800 Subject: [PATCH 07/10] Increase default memory --- .../primeseq/pipeline/SequenceJobResourceAllocator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index b7921c6a..cb4b2ab5 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -92,7 +92,7 @@ private int getAlignerIndexMem(PipelineJob job) String aligner = params.get("alignment"); if (Arrays.asList("BWA-Mem", "BWA-Mem2", "STAR").contains(aligner)) { - return 72; + return 128; } } } @@ -102,7 +102,7 @@ else if (job.getClass().getName().endsWith("ReferenceLibraryPipelineJob")) return 72; } - return 36; + return 72; } @Override From 147738c1fde6d717cb14244b623e17a8e2801217 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 21 Nov 2025 11:06:09 -0800 Subject: [PATCH 08/10] Bugfix to PerformManualIdrStepsTask --- .../org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java index abb8aae2..0507991b 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java @@ -197,7 +197,7 @@ private void updateAnchorDates(PipelineJob pipelineJob, String eventType, String } sourceRecords.get(id).add(date); - if (!existingRecords.containsKey(id) | !existingRecords.get(id).contains(date)) + if (!sourceRecords.containsKey(id) || !sourceRecords.get(id).contains(date)) { toInsert.add(new CaseInsensitiveHashMap<>(Map.of( "subjectId", id, From 477bcb929521bab751d1063366d80900a85e7be9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 21 Nov 2025 11:33:55 -0800 Subject: [PATCH 09/10] Bugfix to PerformManualIdrStepsTask --- .../org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java index 0507991b..8ef3bcb9 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java @@ -183,7 +183,7 @@ private void updateAnchorDates(PipelineJob pipelineJob, String eventType, String final Map> sourceRecords = new HashMap<>(); final List> toInsert = new ArrayList<>(); - new TableSelector(treatments, PageFlowUtil.set("Id", "date", "objectId"), new SimpleFilter(FieldKey.fromString("category"), treatmentCategory), null).forEachResults(rs -> { + new TableSelector(treatments, PageFlowUtil.set("Id", sourceDateField, "objectId"), new SimpleFilter(FieldKey.fromString("category"), treatmentCategory), null).forEachResults(rs -> { String id = rs.getString(FieldKey.fromString("Id")); Date date = rs.getDate(FieldKey.fromString(sourceDateField)); if (date == null) From 9d2203cc4bda956c492b2dc23d6bc3cdd86e4114 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 21 Nov 2025 14:38:41 -0800 Subject: [PATCH 10/10] Update view --- .../resources/queries/study/demographics/Expanded.qview.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml index b1db0a01..9e3d0a62 100644 --- a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml @@ -6,13 +6,14 @@ - + + - +