diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml index 909ee84e..f686476f 100644 --- a/SivStudies/resources/etls/idr-data.xml +++ b/SivStudies/resources/etls/idr-data.xml @@ -100,8 +100,8 @@ - - + + @@ -155,6 +155,11 @@ + + + + + diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml index 1598334f..9e3d0a62 100644 --- a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml @@ -6,11 +6,14 @@ + + - + + diff --git a/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml b/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml index 30c79c60..8f20b4a7 100644 --- a/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml @@ -9,8 +9,6 @@ - - diff --git a/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml b/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml new file mode 100644 index 00000000..43f68638 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml @@ -0,0 +1,22 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml index 135d018b..db95fd32 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml @@ -10,12 +10,31 @@ SIV Infection + /query/executeQuery.view?schemaName=study&query.queryName=treatments&query.Id~eq=${Id}&query.category~eq=SIV Infection + _blank ART + /query/executeQuery.view?schemaName=study&query.queryName=treatments&query.Id~eq=${Id}&query.category~eq=ART + _blank Infection Date + Date + + + ART Initiation + Date + + + ART Initiation (DPI) + + + ART Release + Date + + + ART Release (WPI) allInfections diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index 4e4f9fc7..e9ed1f28 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -1,17 +1,35 @@ SELECT - t.Id, - group_concat(DISTINCT CASE - WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') - ELSE NULL - END, char(10)) as allInfections, - group_concat(DISTINCT CASE - WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') - ELSE NULL - END, char(10)) as allART, - min(CASE - WHEN t.category = 'SIV Infection' THEN t.date - ELSE NULL - END) as infectionDate, + t.*, + TIMESTAMPDIFF('SQL_TSI_WEEK', t.infectionDate, t.artReleaseDate) as artReleaseWPI +FROM ( + SELECT + t.Id, + group_concat(DISTINCT CASE + WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + ELSE NULL + END, char(10)) as allInfections, + min(floor(age(t.DataSets.Demographics.birth, CASE WHEN t.category = 'SIV Infection' THEN t.date ELSE NULL END))) AS ageAtInfection, -FROM study.treatments t -GROUP BY t.Id \ No newline at end of file + group_concat(DISTINCT CASE + WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + ELSE NULL + END, char(10)) as allART, + min(CASE + WHEN t.category = 'SIV Infection' THEN t.date + ELSE NULL + END) as infectionDate, + min(CASE + WHEN t.category = 'ART' THEN t.date + ELSE NULL + END) as artInitiationDate, + min(CASE + WHEN t.category = 'ART' THEN t.timePostSivChallenge.daysPostInfection + ELSE NULL + END) as artInitiationDPI, + min(CASE + WHEN t.category = 'ART' THEN t.artInformation.artRelease + ELSE NULL + END) as artReleaseDate + FROM study.treatments t + GROUP BY t.Id +) t \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsImmunizations.query.xml b/SivStudies/resources/queries/study/demographicsImmunizations.query.xml index 20e1fe98..bd3587c5 100644 --- a/SivStudies/resources/queries/study/demographicsImmunizations.query.xml +++ b/SivStudies/resources/queries/study/demographicsImmunizations.query.xml @@ -10,9 +10,13 @@ Immunizations + /query/executeQuery.view?schemaName=study&query.queryName=immunizations&query.Id~eq=${Id} + _blank Immunization Types + /query/executeQuery.view?schemaName=study&query.queryName=immunizations&query.Id~eq=${Id} + _blank immunizations diff --git a/SivStudies/resources/queries/study/demographicsImmunizations.sql b/SivStudies/resources/queries/study/demographicsImmunizations.sql index a31a58ce..4f28d89b 100644 --- a/SivStudies/resources/queries/study/demographicsImmunizations.sql +++ b/SivStudies/resources/queries/study/demographicsImmunizations.sql @@ -1,6 +1,6 @@ SELECT s.Id, - group_concat(DISTINCT s.treatment, char(10)) as immunizations, + group_concat(DISTINCT COALESCE(s.backbone, s.treatment), char(10)) as immunizations, group_concat(DISTINCT s.category, char(10)) as immunizationTypes, FROM study.immunizations s diff --git a/SivStudies/resources/queries/study/demographicsInterventions.query.xml b/SivStudies/resources/queries/study/demographicsInterventions.query.xml new file mode 100644 index 00000000..c41a1cc7 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsInterventions.query.xml @@ -0,0 +1,30 @@ + + + + + Interventions/Treatments Summary + + + true + true + + + Interventions/Treatments + /query/executeQuery.view?schemaName=study&query.queryName=treatments&query.Id~eq=${Id}&query.category~eq=Intervention + _blank + + + Date of First Intervention + + + First Intervention (DPI) + + + First Intervention (WPI) + + + allInterventions +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsInterventions.sql b/SivStudies/resources/queries/study/demographicsInterventions.sql new file mode 100644 index 00000000..306ac17a --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsInterventions.sql @@ -0,0 +1,20 @@ +SELECT + t.Id, + group_concat(DISTINCT CASE + WHEN t.category = 'Intervention' THEN (t.treatment || ' (' || t.timePostSivChallenge.timePostInfection || ')') + ELSE NULL + END, char(10)) as allInterventions, + min(CASE + WHEN t.category = 'Intervention' THEN t.date + ELSE NULL + END) as firstInterventionDate, + min(CASE + WHEN t.category = 'Intervention' THEN t.timePostSivChallenge.daysPostInfection + ELSE NULL + END) as firstInterventionDPI, + min(CASE + WHEN t.category = 'Intervention' THEN t.timePostSivChallenge.weeksPostInfection + ELSE NULL + END) as firstInterventionWPI +FROM study.treatments t +GROUP BY t.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsPVL.query.xml b/SivStudies/resources/queries/study/demographicsPVL.query.xml new file mode 100644 index 00000000..b097b00a --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsPVL.query.xml @@ -0,0 +1,65 @@ + + + + + PVL Summary + + + true + true + + + SIV Challenge + Date + + + # PVLs + /query/executeQuery.view?schemaName=study&query.queryName=viralLoads&query.Id~eq=${Id}&query.sampleType~eq=Plasma&query.target~eq=SIV + _blank + + + First PVL + + + Last PVL + + + DPI of First PVL + + + DPI of Last PVL + + + WPI of First PVL + + + WPI of Last PVL + + + ART Release + Date + + + + # PVLs Post-ART Release + /query/executeQuery.view?schemaName=study&query.queryName=viralLoads&query.Id~eq=${Id}&query.sampleType~eq=Plasma&query.target~eq=SIV&query.date~dategte=${artRelease} + _blank + + + First PVL Post-ART Release (Weeks) + + + Last PVL Post-ART Release (Weeks) + + + First PVL Post-ART Release (Months) + + + Last PVL Post-ART Release (Months) + + + numPVL +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsPVL.sql b/SivStudies/resources/queries/study/demographicsPVL.sql new file mode 100644 index 00000000..e83aa354 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsPVL.sql @@ -0,0 +1,32 @@ +SELECT + t.Id, + count(*) AS numPVL, + min(sivChallenge) as sivChallenge, + min(t.date) as dateOfFirstPvl, + max(t.date) as dateOfLastPvl, + + min(CONVERT(CASE WHEN t.sivChallenge IS NULL THEN NULL WHEN t.date <= t.sivChallenge THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_DAY', t.sivChallenge, t.date) END, INTEGER)) as firstPvlDPI, + max(CONVERT(CASE WHEN t.sivChallenge IS NULL THEN NULL WHEN t.date <= t.sivChallenge THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_DAY', t.sivChallenge, t.date) END, INTEGER)) as lastPvlDPI, + + min(CONVERT(CASE WHEN t.sivChallenge IS NULL THEN NULL WHEN t.date <= t.sivChallenge THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_WEEK', t.sivChallenge, t.date) END, INTEGER)) as firstPvlWPI, + max(CONVERT(CASE WHEN t.sivChallenge IS NULL THEN NULL WHEN t.date <= t.sivChallenge THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_WEEK', t.sivChallenge, t.date) END, INTEGER)) as lastPvlWPI, + + min(artRelease) as artRelease, + sum(CASE WHEN (t.artRelease IS NOT NULL AND t.date > t.artRelease) THEN 1 ELSE 0 END) as numPVLPostArtRelease, + + min(CONVERT(CASE WHEN t.artRelease IS NULL THEN NULL WHEN t.date <= t.artRelease THEN NULL ELSE age_in_months(t.artRelease, t.date) END, FLOAT)) as firstPvlPostArtReleaseMonths, + max(CONVERT(CASE WHEN t.artRelease IS NULL THEN NULL WHEN t.date <= t.artRelease THEN NULL ELSE age_in_months(t.artRelease, t.date) END, FLOAT)) as lastPvlPostArtReleaseMonths, + + min(CONVERT(CASE WHEN t.artRelease IS NULL THEN NULL WHEN t.date <= t.artRelease THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_WEEK', t.artRelease, t.date) END, INTEGER)) as firstPvlPostArtReleaseWeeks, + max(CONVERT(CASE WHEN t.artRelease IS NULL THEN NULL WHEN t.date <= t.artRelease THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_WEEK', t.artRelease, t.date) END, INTEGER)) as lastPvlPostArtReleaseWeeks + +FROM (SELECT + vl.Id, + vl.date, + (SELECT min(tr.date) as sivChallenge FROM study.treatments tr WHERE tr.category = 'SIV Infection' AND tr.Id = vl.Id) as sivChallenge, + (SELECT max(tr.enddate) as artRelease FROM study.treatments tr WHERE tr.category = 'ART' AND tr.Id = vl.Id) as artRelease + +FROM study.viralLoads vl +WHERE vl.target = 'SIV' AND vl.sampleType = 'Plasma' +) t +GROUP BY t.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsProjects.query.xml b/SivStudies/resources/queries/study/demographicsProjects.query.xml index ec3dddc0..40389751 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.query.xml +++ b/SivStudies/resources/queries/study/demographicsProjects.query.xml @@ -17,12 +17,6 @@ Subgroups/Treatments - - RhCMV Vaccines? - - - SIV/ART Projects? - categories diff --git a/SivStudies/resources/queries/study/demographicsProjects.sql b/SivStudies/resources/queries/study/demographicsProjects.sql index 853db7d1..d1a8689e 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.sql +++ b/SivStudies/resources/queries/study/demographicsProjects.sql @@ -2,10 +2,9 @@ SELECT s.Id, count(s.Id) as totalProjects, group_concat(DISTINCT s.study, char(10)) as allStudies, + group_concat(DISTINCT s.cohortId.studyId.description, char(10)) as studyDescription, group_concat(DISTINCT s.category, char(10)) as categories, - group_concat(DISTINCT s.subgroup, char(10)) as subgroups, + group_concat(DISTINCT s.subgroup, char(10)) as subgroups - GROUP_CONCAT(distinct CASE WHEN s.category = 'RhCMV-Vaccines' THEN 'Yes' ELSE null END, char(10)) as rhCmvVaccines, - GROUP_CONCAT(distinct CASE WHEN s.category = 'SIV/ART' THEN 'Yes' ELSE null END, char(10)) as sivArt FROM study.assignment s GROUP BY s.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/flow.query.xml b/SivStudies/resources/queries/study/flow.query.xml index 63a7946a..3ca630fe 100644 --- a/SivStudies/resources/queries/study/flow.query.xml +++ b/SivStudies/resources/queries/study/flow.query.xml @@ -9,6 +9,9 @@ Start Date Date + + Tissue + Sample Type @@ -18,8 +21,14 @@ Population - - Result + + Parent Population + + + % of Parent + + + Absolute Count Units diff --git a/SivStudies/resources/queries/study/flow/.qview.xml b/SivStudies/resources/queries/study/flow/.qview.xml index 7c9abe0b..1413e211 100644 --- a/SivStudies/resources/queries/study/flow/.qview.xml +++ b/SivStudies/resources/queries/study/flow/.qview.xml @@ -2,10 +2,13 @@ - + + - + + + diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml index 7e1a7a60..2fc7f169 100644 --- a/SivStudies/resources/queries/study/immunizations.query.xml +++ b/SivStudies/resources/queries/study/immunizations.query.xml @@ -15,6 +15,12 @@ Treatment + + Vector Backbone + + + Antigens + Route @@ -27,6 +33,12 @@ Reason + + Vector ID + + + Is Mock? + Comments textarea diff --git a/SivStudies/resources/queries/study/immunizations/.qview.xml b/SivStudies/resources/queries/study/immunizations/.qview.xml index 575150b0..bc9ac8dc 100644 --- a/SivStudies/resources/queries/study/immunizations/.qview.xml +++ b/SivStudies/resources/queries/study/immunizations/.qview.xml @@ -7,6 +7,10 @@ + + + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index 802e195f..2aaf20ea 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -110,13 +110,18 @@ entityid true - varchar + + varchar + varchar + + varchar + varchar @@ -129,6 +134,12 @@ varchar + + boolean + + + integer + varchar @@ -509,13 +520,22 @@ varchar + + varchar + varchar varchar - + + varchar + + + double + + double diff --git a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java new file mode 100644 index 00000000..8ef3bcb9 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java @@ -0,0 +1,273 @@ +package org.labkey.sivstudies.etl; + +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.writer.ContainerUser; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PerformManualIdrStepsTask implements TaskRefTask +{ + protected ContainerUser _containerUser; + + @Override + public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJobException + { + pruneSivChallenges(pipelineJob); + updateVaccineInformation(pipelineJob); + updateChallengeAnchorDates(pipelineJob); + updateArtInitiationAnchorDates(pipelineJob); + + return new RecordedActionSet(); + } + + private void pruneSivChallenges(PipelineJob pipelineJob) throws PipelineJobException + { + TableInfo ti = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("treatments"); + + Map> existingRecords = new HashMap<>(); + new TableSelector(ti, PageFlowUtil.set("Id", "date", "lsid"), new SimpleFilter(FieldKey.fromString("treatment"), "SIV - Unknown", CompareType.NEQ_OR_NULL).addCondition(FieldKey.fromString("category"), "SIV Infection"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("Id")); + if (!existingRecords.containsKey(id)) + { + existingRecords.put(id, new HashSet<>()); + } + + existingRecords.get(id).add(rs.getDate(FieldKey.fromString("date"))); + }); + + final List> toDelete = new ArrayList<>(); + new TableSelector(ti, PageFlowUtil.set("Id", "date", "lsid"), new SimpleFilter(FieldKey.fromString("treatment"), "SIV - Unknown"), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("Id")); + if (!existingRecords.containsKey(id)) + { + return; + } + + if (existingRecords.get(id).contains(rs.getDate(FieldKey.fromString("date")))) + { + toDelete.add(new CaseInsensitiveHashMap<>(Map.of("lsid", rs.getString(FieldKey.fromString("lsid"))))); + } + }); + + if (!toDelete.isEmpty()) + { + pipelineJob.getLogger().info("Deleting " + toDelete.size() + " SIV challenge records"); + + try + { + ti.getUpdateService().deleteRows(_containerUser.getUser(), _containerUser.getContainer(), toDelete, null, null); + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | InvalidKeyException e) + { + throw new PipelineJobException(e); + } + } + } + + private void updateVaccineInformation(PipelineJob pipelineJob) throws PipelineJobException + { + TableInfo ti = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("immunizations"); + + final List> toUpdate = new ArrayList<>(); + new TableSelector(ti, PageFlowUtil.set("lsid", "treatment", "backbone", "antigens")).forEachResults(rs -> { + String treatment = rs.getString(FieldKey.fromString("treatment")); + if (treatment == null) + { + return; + } + + String backbone = rs.getString(FieldKey.fromString("backbone")); + Map updatedRow = new HashMap<>(); + + if (backbone != null && backbone.contains("68-1")) + { + if (treatment.contains("miR-142-126")) + { + updatedRow.put("backbone", "68-1 MHC-1A-only"); + } + else if (treatment.contains("miR-126")) + { + updatedRow.put("backbone", "68-1 MHC-E-only"); + } + else if (treatment.contains("miR-142")) + { + updatedRow.put("backbone", "68-1 MHC-II-only"); + } + } + + if (treatment.toUpperCase().contains("MOCK")) + { + updatedRow.put("isMock", true); + } + + if (!updatedRow.isEmpty()) + { + updatedRow.put("lsid", rs.getString(FieldKey.fromString("lsid"))); + toUpdate.add(updatedRow); + } + }); + + if (!toUpdate.isEmpty()) + { + pipelineJob.getLogger().info("Updating " + toUpdate.size() + " immunization records"); + + try + { + BatchValidationException bve = new BatchValidationException(); + + List> oldKeys = toUpdate.stream().map(x -> (Map)new CaseInsensitiveHashMap<>(Map.of("lsid", x.get("lsid")))).toList(); + ti.getUpdateService().updateRows(_containerUser.getUser(), _containerUser.getContainer(), toUpdate, oldKeys, bve, null, null); + + if (bve.hasErrors()) + { + throw bve; + } + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | InvalidKeyException e) + { + throw new PipelineJobException(e); + } + } + } + + private void updateChallengeAnchorDates(PipelineJob pipelineJob) throws PipelineJobException + { + updateAnchorDates(pipelineJob, "SIV Infection", "SIV Infection", "date"); + } + + private void updateArtInitiationAnchorDates(PipelineJob pipelineJob) throws PipelineJobException + { + updateAnchorDates(pipelineJob, "ART Initiation", "ART", "date"); + updateAnchorDates(pipelineJob, "ART End", "ART", "enddate"); + } + + private void updateAnchorDates(PipelineJob pipelineJob, String eventType, String treatmentCategory, String sourceDateField) throws PipelineJobException + { + TableInfo treatments = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("treatments"); + TableInfo ad = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "studies").getTable("subjectAnchorDates"); + + Map> existingRecords = new HashMap<>(); + new TableSelector(ad, PageFlowUtil.set("subjectId", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), eventType), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("subjectId")); + if (!existingRecords.containsKey(id)) + { + existingRecords.put(id, new HashSet<>()); + } + + existingRecords.get(id).add(rs.getDate(FieldKey.fromString("date"))); + }); + + final Map> sourceRecords = new HashMap<>(); + final List> toInsert = new ArrayList<>(); + new TableSelector(treatments, PageFlowUtil.set("Id", sourceDateField, "objectId"), new SimpleFilter(FieldKey.fromString("category"), treatmentCategory), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("Id")); + Date date = rs.getDate(FieldKey.fromString(sourceDateField)); + if (date == null) + { + return; + } + + if (!sourceRecords.containsKey(id)) + { + sourceRecords.put(id, new HashSet<>()); + } + sourceRecords.get(id).add(date); + + if (!sourceRecords.containsKey(id) || !sourceRecords.get(id).contains(date)) + { + toInsert.add(new CaseInsensitiveHashMap<>(Map.of( + "subjectId", id, + "date", date, + "category", eventType, + "sourceRecord", rs.getString(FieldKey.fromString("objectId")) + ))); + } + }); + + if (!toInsert.isEmpty()) + { + pipelineJob.getLogger().info("Inserting " + toInsert.size() + " " + eventType + " anchor date records"); + + try + { + BatchValidationException bve = new BatchValidationException(); + ad.getUpdateService().insertRows(_containerUser.getUser(), _containerUser.getContainer(), toInsert, bve, null, null); + + if (bve.hasErrors()) + { + throw bve; + } + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e) + { + throw new PipelineJobException(e); + } + } + + final List> toDelete = new ArrayList<>(); + new TableSelector(ad, PageFlowUtil.set("subjectId", "date", "rowid"), new SimpleFilter(FieldKey.fromString("eventLabel"), eventType), null).forEachResults(rs -> { + String id = rs.getString(FieldKey.fromString("subjectId")); + Date date = rs.getDate(FieldKey.fromString("date")); + if (!sourceRecords.containsKey(id) | !sourceRecords.get(id).contains(date)) + { + toDelete.add(new CaseInsensitiveHashMap<>(Map.of("rowid", rs.getInt(FieldKey.fromString("rowId"))))); + } + }); + + if (!toDelete.isEmpty()) + { + pipelineJob.getLogger().info("Deleting " + toDelete.size() + " " + eventType + " anchor date records"); + + try + { + ad.getUpdateService().deleteRows(_containerUser.getUser(), _containerUser.getContainer(), toDelete, null, null); + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | InvalidKeyException e) + { + throw new PipelineJobException(e); + } + } + } + + @Override + public List getRequiredSettings() + { + return List.of(); + } + + @Override + public void setSettings(Map map) throws XmlException + { + + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} diff --git a/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java index 00e5b739..3d9fb725 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java +++ b/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; public class AutoCreateDemographicsTrigger extends DefaultDatasetTrigger { @@ -46,55 +47,33 @@ public Factory() } } - private static final String CACHE_KEY = "~~AutoCreateDemographicsTrigger.IdsToCreate~~"; + private final Set _idsToCheck = new CaseInsensitiveHashSet(); @Override protected void afterUpsert(TableInfo table, Container c, User user, @Nullable Map newRow, @Nullable Map oldRow, ValidationException errors, Map extraContext) throws ValidationException { - if (extraContext == null) + String idField = getIdField(c); + String id = newRow.get(idField) != null ? newRow.get(idField).toString() : null; + if (id != null) { - _log.error("extraContext is null in AutoCreateDemographicsTrigger.afterUpsert()"); - return; - } - - if (!extraContext.containsKey(AutoCreateDemographicsTrigger.CACHE_KEY)) - { - extraContext.put(CACHE_KEY, new CaseInsensitiveHashSet()); - } - - if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s) - { - String idField = getIdField(c); - String id = newRow.get(idField) != null ? newRow.get(idField).toString() : null; - if (id != null) - { - s.add(id); - } + _idsToCheck.add(id); } } @Override public void complete(TableInfo table, Container c, User user, TableInfo.TriggerType event, BatchValidationException errors, Map extraContext) { - if (extraContext == null) - { - _log.error("extraContext is null in AutoCreateDemographicsTrigger.complete()"); - return; - } - - if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s) + if (!_idsToCheck.isEmpty()) { - s = new CaseInsensitiveHashSet(s); - String idField = getIdField(c); TableInfo ti = QueryService.get().getUserSchema(user, getTargetContainer(c), "study").getTable("demographics"); - List existingIds = new TableSelector(ti, PageFlowUtil.set(idField), new SimpleFilter(FieldKey.fromString(idField), s, CompareType.IN), null).getArrayList(String.class); + List existingIds = new TableSelector(ti, PageFlowUtil.set(idField), new SimpleFilter(FieldKey.fromString(idField), _idsToCheck, CompareType.IN), null).getArrayList(String.class); - s.removeAll(existingIds); + _idsToCheck.removeAll(existingIds); - if (!s.isEmpty()) + if (!_idField.isEmpty()) { - List> toInsert = s.stream().map(id -> Map.of(idField, (Object)id)).toList(); + List> toInsert = _idsToCheck.stream().map(id -> Map.of(idField, (Object)id)).toList(); try { ti.getUpdateService().insertRows(user, c, toInsert, null, null, null); diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 1fc15ddb..8237763c 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -13,6 +13,7 @@ import org.labkey.api.data.WrappedColumn; import org.labkey.api.ldk.table.AbstractTableCustomizer; import org.labkey.api.query.ExprColumn; +import org.labkey.api.query.FieldKey; import org.labkey.api.query.LookupForeignKey; import org.labkey.api.query.QueryDefinition; import org.labkey.api.query.QueryException; @@ -27,6 +28,7 @@ import org.labkey.api.util.logging.LogHelper; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -251,6 +253,13 @@ private void appendDemographicsColumns(AbstractTableInfo parentTable) parentTable.addColumn(colInfo); } + if (parentTable.getColumn("interventions") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(parentTable.getUserSchema(), "demographicsInterventions", parentTable, "interventions"); + colInfo.setLabel("Interventions"); + parentTable.addColumn(colInfo); + } + if (parentTable.getColumn("outcomes") == null) { BaseColumnInfo colInfo = getWrappedIdCol(parentTable.getUserSchema(), "demographicsOutcomes", parentTable, "outcomes"); @@ -264,6 +273,13 @@ private void appendDemographicsColumns(AbstractTableInfo parentTable) colInfo.setLabel("SIV/ART Dates"); parentTable.addColumn(colInfo); } + + if (parentTable.getColumn("pvlInfo") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(parentTable.getUserSchema(), "demographicsPVL", parentTable, "pvlInfo"); + colInfo.setLabel("PVL Info"); + parentTable.addColumn(colInfo); + } } private void appendPvlColumns(DatasetTable ds, String subjectColName, String dateColName) @@ -348,17 +364,25 @@ public TableInfo getLookupTableInfo() String name = queryName + "_sivChallenge"; UserSchema targetSchema = targetTable.getUserSchema().getDefaultSchema().getUserSchema(targetSchemaName); QueryDefinition qd = QueryService.get().createQueryDef(u, targetSchemaContainer, targetSchema, name); - qd.setSql("SELECT\n" + + qd.setSql("SELECT t.*,\n" + + "CASE\n" + + "WHEN t.daysPostInfection IS NULL THEN NULL\n" + + "WHEN t.daysPostInfection <= 28 THEN (CAST(t.daysPostInfection AS VARCHAR) || ' DPI')\n" + + "ELSE (CAST(t.weeksPostInfection AS VARCHAR) || ' WPI')\n" + + "END as timePostInfection\n" + + "FROM (SELECT\n" + "max(ad.date) as infectionDate,\n" + // NOTE: CAST() is used to ensure whole numbers "CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), INTEGER) as daysPostInfection,\n" + - "CONVERT(age_in_months(CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), FLOAT) as monthsPostInfection,\n" + + "CONVERT((CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), INTEGER) / 7), INTEGER) as weeksPostInfection,\n" + + "ROUND(CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), DOUBLE) / 7.0, 1) as weeksPostInfectionDecimal,\n" + + "CONVERT(age_in_months(CAST(max(ad.date) AS DATE), CAST(c." + dateColName + " AS DATE)), DOUBLE) as monthsPostInfection,\n" + "c." + pkCol.getFieldKey().toString() + "\n" + "FROM \"" + schemaName + "\".\"" + queryName + "\" c " + "JOIN studies.subjectAnchorDates ad ON (ad.subjectId = c." + idCol.getFieldKey().toSQLString() + ")\n" + "WHERE ad.eventLabel = 'SIV Infection'\n" + "GROUP BY c.date, c." + pkCol.getFieldKey().toString() + "\n" + - "HAVING count(*) = 1" + "HAVING count(*) = 1) t" ); qd.setIsTemporary(true); @@ -380,7 +404,12 @@ public TableInfo getLookupTableInfo() ((BaseColumnInfo)ti.getColumn("infectionDate")).setLabel("Infection Date"); ((BaseColumnInfo)ti.getColumn("daysPostInfection")).setLabel("Days Post-Infection"); + ((BaseColumnInfo)ti.getColumn("weeksPostInfection")).setLabel("Weeks Post-Infection"); ((BaseColumnInfo)ti.getColumn("monthsPostInfection")).setLabel("Months Post-Infection"); + + BaseColumnInfo tpi = ((BaseColumnInfo)ti.getColumn("timePostInfection")); + tpi.setLabel("Time Post-Infection"); + tpi.setSortFieldKeys(Arrays.asList(FieldKey.fromString("daysPostInfection"))); } return ti; diff --git a/mcc/resources/queries/study/demographicsLittermates.sql b/mcc/resources/queries/study/demographicsLittermates.sql index 0b3f411d..d2edb072 100644 --- a/mcc/resources/queries/study/demographicsLittermates.sql +++ b/mcc/resources/queries/study/demographicsLittermates.sql @@ -1,16 +1,10 @@ SELECT -d1.Id, -d1.litterId, + d1.Id, + d1.litterId, -GROUP_CONCAT(distinct d2.Id, ',') as litterMates + (SELECT GROUP_CONCAT(distinct d2.Id, ',') as litterMates FROM study.Demographics d2 WHERE d2.qcstate.publicdata = true AND d1.litterId = d2.litterId AND d1.id != d2.id) as litterMates FROM study.Demographics d1 -JOIN study.Demographics d2 ON (d1.litterId = d2.litterId AND d1.id != d2.id) - -WHERE - d1.qcstate.publicdata = true AND - d2.qcstate.publicdata = true - -GROUP BY d1.Id, d1.litterId \ No newline at end of file +WHERE d1.qcstate.publicdata = true diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index b7921c6a..cb4b2ab5 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -92,7 +92,7 @@ private int getAlignerIndexMem(PipelineJob job) String aligner = params.get("alignment"); if (Arrays.asList("BWA-Mem", "BWA-Mem2", "STAR").contains(aligner)) { - return 72; + return 128; } } } @@ -102,7 +102,7 @@ else if (job.getClass().getName().endsWith("ReferenceLibraryPipelineJob")) return 72; } - return 36; + return 72; } @Override