From 25e7206cb835b14f0b1c359041411aa84dfe3762 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 25 Nov 2025 09:08:20 -0800 Subject: [PATCH 01/14] Expand SivStudiesDataValidationNotification --- .../SivStudiesDataValidationNotification.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java index 24c03dbb..9e2c05ab 100644 --- a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java +++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java @@ -76,6 +76,7 @@ public String getEmailSubject(Container c) infectionAnchorDateDiscordance(c, u, msg); pvlWithoutInfectionDate(c, u, msg); idsMissingFromDemographics(c, u, msg); + missingArtRecord(c, u, msg); if (!msg.isEmpty()) { @@ -148,6 +149,20 @@ private void idsMissingFromDemographics(Container c, User u, StringBuilder msg) genericQueryCheck(c, u, msg, "study", s.getSubjectNounSingular(), "IDs with data in the study not present in the demographics table", filter); } + private void missingArtRecord(Container c, User u, StringBuilder msg) + { + Study s = StudyService.get().getStudy(getTargetContainer(c)); + if (s == null) + { + return; + } + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("projects/studyDescription"), "ART", CompareType.CONTAINS); + filter.addCondition(FieldKey.fromString("projects/studyDescription"), "No ART", CompareType.DOES_NOT_CONTAIN); + filter.addCondition(FieldKey.fromString("sivART/artInitiationDPI"), null, CompareType.ISBLANK); + genericQueryCheck(c, u, msg, "study", s.getSubjectNounSingular(), "IDs assigned to an ART study without a record of ART", filter); + } + protected Container getTargetContainer(Container c) { return c.isWorkbookOrTab() ? c.getParent() : c; From 60fe44605ea49e61ca14a49201c530560f33964a Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 26 Nov 2025 11:07:09 -0800 Subject: [PATCH 02/14] Expand SIV study queries --- .../Expanded ART Detail.qview.xml | 24 +++++++++++++++++++ .../study/demographics/Expanded.qview.xml | 2 -- .../demographics/SIV and ART Info.qview.xml | 2 +- .../study/demographicsChallengeAndArt.sql | 2 +- .../study/demographicsInterventions.query.xml | 3 +++ .../study/demographicsInterventions.sql | 22 +++++------------ .../study/demographicsProjects.query.xml | 8 ++++++- .../queries/study/demographicsProjects.sql | 4 +++- .../treatments/With SIV-ART Dates.qview.xml | 23 ++++++++++++++++++ 9 files changed, 68 insertions(+), 22 deletions(-) create mode 100644 SivStudies/resources/queries/study/demographics/Expanded ART Detail.qview.xml create mode 100644 SivStudies/resources/queries/study/treatments/With SIV-ART Dates.qview.xml diff --git a/SivStudies/resources/queries/study/demographics/Expanded ART Detail.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded ART Detail.qview.xml new file mode 100644 index 00000000..92e29c31 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/Expanded ART Detail.qview.xml @@ -0,0 +1,24 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml index 9e3d0a62..9ac5f0b1 100644 --- a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml @@ -4,8 +4,6 @@ - - diff --git a/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml b/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml index 43f68638..028b3a7b 100644 --- a/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml +++ b/SivStudies/resources/queries/study/demographics/SIV and ART Info.qview.xml @@ -4,7 +4,7 @@ - + diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index e9ed1f28..69d85950 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -5,7 +5,7 @@ FROM ( SELECT t.Id, group_concat(DISTINCT CASE - WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + WHEN t.category = 'SIV Infection' THEN (t.treatment || (CASE WHEN t.route IS NULL then '' ELSE (', ' || t.route) END) || (CASE WHEN t.amount IS NULL then '' ELSE (', ' || CAST(t.amount as VARCHAR) || ' ' || t.amount_units) END)) ELSE NULL END, char(10)) as allInfections, min(floor(age(t.DataSets.Demographics.birth, CASE WHEN t.category = 'SIV Infection' THEN t.date ELSE NULL END))) AS ageAtInfection, diff --git a/SivStudies/resources/queries/study/demographicsInterventions.query.xml b/SivStudies/resources/queries/study/demographicsInterventions.query.xml index c41a1cc7..b717f3a3 100644 --- a/SivStudies/resources/queries/study/demographicsInterventions.query.xml +++ b/SivStudies/resources/queries/study/demographicsInterventions.query.xml @@ -22,6 +22,9 @@ First Intervention (WPI) + + First Intervention Relative to ART Release (Weeks) + allInterventions diff --git a/SivStudies/resources/queries/study/demographicsInterventions.sql b/SivStudies/resources/queries/study/demographicsInterventions.sql index 306ac17a..758ebd7f 100644 --- a/SivStudies/resources/queries/study/demographicsInterventions.sql +++ b/SivStudies/resources/queries/study/demographicsInterventions.sql @@ -1,20 +1,10 @@ SELECT t.Id, - group_concat(DISTINCT CASE - WHEN t.category = 'Intervention' THEN (t.treatment || ' (' || t.timePostSivChallenge.timePostInfection || ')') - ELSE NULL - END, char(10)) as allInterventions, - min(CASE - WHEN t.category = 'Intervention' THEN t.date - ELSE NULL - END) as firstInterventionDate, - min(CASE - WHEN t.category = 'Intervention' THEN t.timePostSivChallenge.daysPostInfection - ELSE NULL - END) as firstInterventionDPI, - min(CASE - WHEN t.category = 'Intervention' THEN t.timePostSivChallenge.weeksPostInfection - ELSE NULL - END) as firstInterventionWPI + group_concat(DISTINCT (t.treatment || ' (' || t.timePostSivChallenge.timePostInfection || ')'), char(10)) as allInterventions, + min(t.date) as firstInterventionDate, + min(t.timePostSivChallenge.daysPostInfection) as firstInterventionDPI, + min(t.timePostSivChallenge.weeksPostInfection) as firstInterventionWPI, + min(t.timePostSivChallenge.weeksPostInfection) - min(t.sivART.artReleaseWPI) as firstInterventionPostArtReleaseWeeks FROM study.treatments t +WHERE t.category = 'Intervention' GROUP BY t.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsProjects.query.xml b/SivStudies/resources/queries/study/demographicsProjects.query.xml index 40389751..c8047114 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.query.xml +++ b/SivStudies/resources/queries/study/demographicsProjects.query.xml @@ -12,7 +12,13 @@ Study Categories - All Studies + Studies + + + Studies and Analyses + + + Analyses Subgroups/Treatments diff --git a/SivStudies/resources/queries/study/demographicsProjects.sql b/SivStudies/resources/queries/study/demographicsProjects.sql index d1a8689e..b54d1591 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.sql +++ b/SivStudies/resources/queries/study/demographicsProjects.sql @@ -1,7 +1,9 @@ SELECT s.Id, count(s.Id) as totalProjects, - group_concat(DISTINCT s.study, char(10)) as allStudies, + group_concat(DISTINCT s.study, char(10)) as allStudiesAndAnalyses, + group_concat(DISTINCT CASE WHEN s.category = 'Analysis Cohorts' THEN NULL ELSE s.study END, char(10)) as allStudies, + group_concat(DISTINCT CASE WHEN s.category = 'Analysis Cohorts' THEN s.study ELSE NULL END, char(10)) as analysisGroups, group_concat(DISTINCT s.cohortId.studyId.description, char(10)) as studyDescription, group_concat(DISTINCT s.category, char(10)) as categories, group_concat(DISTINCT s.subgroup, char(10)) as subgroups diff --git a/SivStudies/resources/queries/study/treatments/With SIV-ART Dates.qview.xml b/SivStudies/resources/queries/study/treatments/With SIV-ART Dates.qview.xml new file mode 100644 index 00000000..087c2d4f --- /dev/null +++ b/SivStudies/resources/queries/study/treatments/With SIV-ART Dates.qview.xml @@ -0,0 +1,23 @@ + \ No newline at end of file From 159012593dd33117f755ad6bf185eba9404f486f Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 26 Nov 2025 13:28:58 -0800 Subject: [PATCH 03/14] Another round of SIV study formatting --- .../queries/study/demographics/Expanded ART Detail.qview.xml | 1 + .../queries/study/demographicsChallengeAndArt.query.xml | 4 +++- .../resources/queries/study/demographicsChallengeAndArt.sql | 2 +- .../resources/queries/study/demographicsProjects.query.xml | 5 ++++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/SivStudies/resources/queries/study/demographics/Expanded ART Detail.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded ART Detail.qview.xml index 92e29c31..d74baa9e 100644 --- a/SivStudies/resources/queries/study/demographics/Expanded ART Detail.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Expanded ART Detail.qview.xml @@ -9,6 +9,7 @@ + diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml index db95fd32..c0e5e43a 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml @@ -12,11 +12,13 @@ SIV Infection /query/executeQuery.view?schemaName=study&query.queryName=treatments&query.Id~eq=${Id}&query.category~eq=SIV Infection _blank + 150 - ART + ART Regimen /query/executeQuery.view?schemaName=study&query.queryName=treatments&query.Id~eq=${Id}&query.category~eq=ART _blank + 175 Infection Date diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index 69d85950..408088ed 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -11,7 +11,7 @@ FROM ( min(floor(age(t.DataSets.Demographics.birth, CASE WHEN t.category = 'SIV Infection' THEN t.date ELSE NULL END))) AS ageAtInfection, group_concat(DISTINCT CASE - WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + WHEN t.category = 'ART' THEN (t.treatment || ' (' || t.timePostSivChallenge.timePostInfection || ')') ELSE NULL END, char(10)) as allART, min(CASE diff --git a/SivStudies/resources/queries/study/demographicsProjects.query.xml b/SivStudies/resources/queries/study/demographicsProjects.query.xml index c8047114..11e8665c 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.query.xml +++ b/SivStudies/resources/queries/study/demographicsProjects.query.xml @@ -13,12 +13,15 @@ Studies + /query/executeQuery.view?schemaName=study&query.queryName=assignment&query.Id~eq=${Id} + _blank Studies and Analyses - Analyses + Analysis Groups + 150 Subgroups/Treatments From 67c66bf7db0d82c45e98f71fcc2b60b46af3ac09 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 26 Nov 2025 15:53:07 -0800 Subject: [PATCH 04/14] More query improvements --- .../study/demographicsChallengeAndArt.sql | 2 +- .../study/demographicsInterventions.sql | 2 +- .../study/missingAnchorDates.query.xml | 9 ++++++ .../queries/study/missingAnchorDates.sql | 30 +++++++++++++++++++ .../query/SivStudiesCustomizer.java | 5 ++-- 5 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 SivStudies/resources/queries/study/missingAnchorDates.query.xml create mode 100644 SivStudies/resources/queries/study/missingAnchorDates.sql diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index 408088ed..6805ca6d 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -11,7 +11,7 @@ FROM ( min(floor(age(t.DataSets.Demographics.birth, CASE WHEN t.category = 'SIV Infection' THEN t.date ELSE NULL END))) AS ageAtInfection, group_concat(DISTINCT CASE - WHEN t.category = 'ART' THEN (t.treatment || ' (' || t.timePostSivChallenge.timePostInfection || ')') + WHEN t.category = 'ART' THEN (t.treatment || ' (' || COALESCE(t.timePostSivChallenge.timePostInfection, 'Unk DPI') || ')') ELSE NULL END, char(10)) as allART, min(CASE diff --git a/SivStudies/resources/queries/study/demographicsInterventions.sql b/SivStudies/resources/queries/study/demographicsInterventions.sql index 758ebd7f..7d1c77da 100644 --- a/SivStudies/resources/queries/study/demographicsInterventions.sql +++ b/SivStudies/resources/queries/study/demographicsInterventions.sql @@ -1,6 +1,6 @@ SELECT t.Id, - group_concat(DISTINCT (t.treatment || ' (' || t.timePostSivChallenge.timePostInfection || ')'), char(10)) as allInterventions, + group_concat(DISTINCT (t.treatment || ' (' || COALESCE(t.timePostSivChallenge.timePostInfection, 'Unk DPI') || ')'), char(10)) as allInterventions, min(t.date) as firstInterventionDate, min(t.timePostSivChallenge.daysPostInfection) as firstInterventionDPI, min(t.timePostSivChallenge.weeksPostInfection) as firstInterventionWPI, diff --git a/SivStudies/resources/queries/study/missingAnchorDates.query.xml b/SivStudies/resources/queries/study/missingAnchorDates.query.xml new file mode 100644 index 00000000..78b2805f --- /dev/null +++ b/SivStudies/resources/queries/study/missingAnchorDates.query.xml @@ -0,0 +1,9 @@ + + + + + Missing Anchor Dates +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/missingAnchorDates.sql b/SivStudies/resources/queries/study/missingAnchorDates.sql new file mode 100644 index 00000000..ba56cacf --- /dev/null +++ b/SivStudies/resources/queries/study/missingAnchorDates.sql @@ -0,0 +1,30 @@ +SELECT + dc.Id as SubjectId, + dc.infectionDate as date, + 'SIV Infection' as eventLabel + +FROM study.demographicsChallengeAndArt dc +LEFT JOIN studies.subjectAnchorDates ad ON (dc.Id = ad.SubjectId AND dc.infectionDate = ad.date AND ad.eventLabel = 'SIV Infection') +WHERE ad.rowid IS NULL AND dc.infectionDate IS NOT NULL + +UNION ALL + +SELECT + dc.Id as SubjectId, + dc.artInitiationDate as date, + 'ART Initiation' as eventLabel + +FROM study.demographicsChallengeAndArt dc + LEFT JOIN studies.subjectAnchorDates ad ON (dc.Id = ad.SubjectId AND dc.artInitiationDate = ad.date AND ad.eventLabel = 'ART Initiation') +WHERE ad.rowid IS NULL AND dc.artInitiationDate IS NOT NULL + +UNION ALL + +SELECT + dc.Id as SubjectId, + dc.artReleaseDate as date, + 'ART Release' as eventLabel + +FROM study.demographicsChallengeAndArt dc + LEFT JOIN studies.subjectAnchorDates ad ON (dc.Id = ad.SubjectId AND dc.artReleaseDate = ad.date AND ad.eventLabel = 'ART Release') +WHERE ad.rowid IS NULL AND dc.artReleaseDate IS NOT NULL \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 8237763c..0f089490 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -382,7 +382,7 @@ public TableInfo getLookupTableInfo() "JOIN studies.subjectAnchorDates ad ON (ad.subjectId = c." + idCol.getFieldKey().toSQLString() + ")\n" + "WHERE ad.eventLabel = 'SIV Infection'\n" + "GROUP BY c.date, c." + pkCol.getFieldKey().toString() + "\n" + - "HAVING count(*) = 1) t" + "HAVING count(DISTINCT c.date) = 1) t" ); qd.setIsTemporary(true); @@ -472,9 +472,10 @@ public TableInfo getLookupTableInfo() "GROUP_CONCAT(DISTINCT tr.treatment) AS artTreatment,\n" + "c." + pkCol.getFieldKey().toString() + "\n" + "FROM \"" + schemaName + "\".\"" + queryName + "\" c " + + // TODO: consider whether this should include all dates "JOIN study.treatments tr ON (tr.category = 'ART' AND CAST(tr.date AS DATE) <= CAST(c." + dateCol.getFieldKey().toString() + " AS DATE) AND tr.Id = c." + idCol.getFieldKey().toSQLString() + ")\n" + "GROUP BY c." + dateCol.getFieldKey().toString() + ", c." + pkCol.getFieldKey().toString() + "\n" + - "HAVING COUNT(*) = 1" + "HAVING COUNT(DISTINCT tr.date) = 1" ); qd.setIsTemporary(true); From bb108ecd24928570d35010c8efe131f1c06d5283 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 26 Nov 2025 15:55:21 -0800 Subject: [PATCH 05/14] Expand SivStudiesDataValidationNotification --- .../SivStudiesDataValidationNotification.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java index 9e2c05ab..d3109e27 100644 --- a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java +++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java @@ -77,6 +77,7 @@ public String getEmailSubject(Container c) pvlWithoutInfectionDate(c, u, msg); idsMissingFromDemographics(c, u, msg); missingArtRecord(c, u, msg); + missingAnchorDates(c, u, msg); if (!msg.isEmpty()) { @@ -163,6 +164,17 @@ private void missingArtRecord(Container c, User u, StringBuilder msg) genericQueryCheck(c, u, msg, "study", s.getSubjectNounSingular(), "IDs assigned to an ART study without a record of ART", filter); } + private void missingAnchorDates(Container c, User u, StringBuilder msg) + { + Study s = StudyService.get().getStudy(getTargetContainer(c)); + if (s == null) + { + return; + } + + genericQueryCheck(c, u, msg, "study", "missingAnchorDates", "records missing from the anchor dates table"); + } + protected Container getTargetContainer(Container c) { return c.isWorkbookOrTab() ? c.getParent() : c; From 5fcf8760fb72a9ab93be2c10407b4da8aac775e7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 28 Nov 2025 07:48:09 -0800 Subject: [PATCH 06/14] Improve SQL --- .../queries/study/infectionAnchorDateDiscordance.sql | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql index c19cd094..cb140f34 100644 --- a/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql +++ b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql @@ -1,16 +1,15 @@ SELECT - t.Id, + coalesce(t.Id, t2.subjectId) as Id, 'SIV Infection' as category, t.date as treatmentTableDate, t2.date as subjectAnchorDatesTableDate FROM study.treatments t FULL JOIN studies.subjectAnchorDates t2 ON ( - t.category = 'SIV Infection' AND - t2.subjectId = t.Id AND - t2.eventLabel = 'SIV Infection' + t.category = t2.eventLabel AND + t2.subjectId = t.Id ) -WHERE t2.date != t.date OR +WHERE (t2.date != t.date OR t2.date IS NULL OR - t.date IS NULL \ No newline at end of file + t.date IS NULL) AND COALESCE(t.category, t2.eventLabel) = 'SIV Infection' \ No newline at end of file From 32c78496d87b839a6d28b344b6dc50f44470bb8d Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 28 Nov 2025 08:58:24 -0800 Subject: [PATCH 07/14] Improve ETLs to filter non-SIV data --- IDR/resources/queries/bimber_data/idrAnchorDateSource.sql | 4 +++- IDR/resources/queries/bimber_data/idrOutcomeSource.sql | 3 ++- SivStudies/resources/etls/idr-data.xml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql b/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql index 7bdf019d..22d9ebd9 100644 --- a/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql +++ b/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql @@ -7,6 +7,7 @@ PID0 as date, FROM bimber_data.subjects WHERE PID0 IS NOT NULL +AND Cohort NOT IN ('PC549', 'PC585') AND Cohort NOT LIKE 'W%' UNION ALL @@ -18,4 +19,5 @@ D0 as date, 'Hansen/IDR' as dataSource FROM bimber_data.subjects -WHERE D0 IS NOT NULL \ No newline at end of file +WHERE D0 IS NOT NULL + AND Cohort NOT IN ('PC549', 'PC585') AND Cohort NOT LIKE 'W%' \ No newline at end of file diff --git a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql index d254000c..0196fd26 100644 --- a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql +++ b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql @@ -12,4 +12,5 @@ END as outcome, 'Hansen/IDR' as dataSource FROM bimber_data.subjects -WHERE contprog IS NOT NULL AND contprog != '' \ No newline at end of file +WHERE contprog IS NOT NULL AND contprog != '' +AND Cohort NOT IN ('PC549', 'PC585') AND Cohort NOT LIKE 'W%' \ No newline at end of file diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml index f686476f..949b72b1 100644 --- a/SivStudies/resources/etls/idr-data.xml +++ b/SivStudies/resources/etls/idr-data.xml @@ -125,7 +125,7 @@ - + From eef667384aa7c5597394241b29f7ad19349f5ac6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 28 Nov 2025 09:44:31 -0800 Subject: [PATCH 08/14] Improve ETLs to filter non-SIV data --- IDR/resources/queries/bimber_data/idrAnchorDateSource.sql | 4 ++-- IDR/resources/queries/bimber_data/idrOutcomeSource.sql | 2 +- SivStudies/resources/etls/idr-data.xml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql b/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql index 22d9ebd9..50205baf 100644 --- a/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql +++ b/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql @@ -7,7 +7,7 @@ PID0 as date, FROM bimber_data.subjects WHERE PID0 IS NOT NULL -AND Cohort NOT IN ('PC549', 'PC585') AND Cohort NOT LIKE 'W%' +AND Cohort NOT IN ('PC549', 'PC585', 'PC529') AND Cohort NOT LIKE 'W%' UNION ALL @@ -20,4 +20,4 @@ D0 as date, FROM bimber_data.subjects WHERE D0 IS NOT NULL - AND Cohort NOT IN ('PC549', 'PC585') AND Cohort NOT LIKE 'W%' \ No newline at end of file + AND Cohort NOT IN ('PC549', 'PC585', 'PC529') AND Cohort NOT LIKE 'W%' \ No newline at end of file diff --git a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql index 0196fd26..277eb04e 100644 --- a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql +++ b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql @@ -13,4 +13,4 @@ END as outcome, FROM bimber_data.subjects WHERE contprog IS NOT NULL AND contprog != '' -AND Cohort NOT IN ('PC549', 'PC585') AND Cohort NOT LIKE 'W%' \ No newline at end of file +AND Cohort NOT IN ('PC549', 'PC585', 'PC529') AND Cohort NOT LIKE 'W%' \ No newline at end of file diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml index 949b72b1..a7756e8c 100644 --- a/SivStudies/resources/etls/idr-data.xml +++ b/SivStudies/resources/etls/idr-data.xml @@ -125,7 +125,7 @@ - + From 50d375f5551737bf71562a78ab516f3cae60e7ae Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 28 Nov 2025 11:20:00 -0800 Subject: [PATCH 09/14] Add SIV view --- .../Expanded Vaccine Detail.qview.xml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 SivStudies/resources/queries/study/demographics/Expanded Vaccine Detail.qview.xml diff --git a/SivStudies/resources/queries/study/demographics/Expanded Vaccine Detail.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded Vaccine Detail.qview.xml new file mode 100644 index 00000000..5734403d --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/Expanded Vaccine Detail.qview.xml @@ -0,0 +1,21 @@ + \ No newline at end of file From 389425ac218468f92201a081c21929fd6a335adb Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 28 Nov 2025 21:25:26 -0800 Subject: [PATCH 10/14] Update dependencies --- mcc/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mcc/package-lock.json b/mcc/package-lock.json index 661d6ff8..7550478d 100644 --- a/mcc/package-lock.json +++ b/mcc/package-lock.json @@ -6307,9 +6307,9 @@ "dev": true }, "node_modules/node-forge": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", - "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.2.tgz", + "integrity": "sha512-6xKiQ+cph9KImrRh0VsjH2d8/GXA4FIMlgU4B757iI1ApvcyA9VlouP0yZJha01V+huImO+kKMU7ih+2+E14fw==", "dev": true, "engines": { "node": ">= 6.13.0" From 48d9f7b006e4496b8483e5c31b85cbc5b40dd991 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 29 Nov 2025 08:12:51 -0800 Subject: [PATCH 11/14] Change Dependabot schedule from daily to weekly --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 583decfd..6fddca0d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,4 +4,4 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "daily" \ No newline at end of file + interval: "weekly" From 69618db34b3b03535728bd2504113a458ed4337b Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 29 Nov 2025 09:46:34 -0800 Subject: [PATCH 12/14] Minor code cleanup --- .../org/labkey/primeseq/pipeline/ClusterMaintenanceTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/ClusterMaintenanceTask.java b/primeseq/src/org/labkey/primeseq/pipeline/ClusterMaintenanceTask.java index cbeddc19..f1a00f5e 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/ClusterMaintenanceTask.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/ClusterMaintenanceTask.java @@ -188,7 +188,7 @@ private void inspectFolder(Logger log, File workDirBase) if (!workDirBase.exists()) { - log.error("Unable to find workdir: " + workDirBase.getPath()); + log.warn("Unable to find workdir: " + workDirBase.getPath()); return; } From 2d0b2aeb9e3ea38aa024371e5c83e06429f4ce19 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 1 Dec 2025 22:10:35 -0800 Subject: [PATCH 13/14] Add study summary columns --- .../queries/studies/studies.query.xml | 21 +++++++++ .../queries/studies/studies/.qview.xml | 18 ++++++++ .../queries/studies/studiesSummary.query.xml | 38 ++++++++++++++++ .../queries/studies/studiesSummary.sql | 16 +++++++ .../studies/studyCohortSummary.query.xml | 38 ++++++++++++++++ .../queries/studies/studyCohortSummary.sql | 16 +++++++ .../queries/studies/studyCohorts.query.xml | 21 +++++++++ .../queries/studies/studyCohorts/.qview.xml | 27 ++++++++++++ .../etl/PerformManualIdrStepsTask.java | 43 +++++++++++++++++++ 9 files changed, 238 insertions(+) create mode 100644 SivStudies/resources/queries/studies/studies.query.xml create mode 100644 SivStudies/resources/queries/studies/studies/.qview.xml create mode 100644 SivStudies/resources/queries/studies/studiesSummary.query.xml create mode 100644 SivStudies/resources/queries/studies/studiesSummary.sql create mode 100644 SivStudies/resources/queries/studies/studyCohortSummary.query.xml create mode 100644 SivStudies/resources/queries/studies/studyCohortSummary.sql create mode 100644 SivStudies/resources/queries/studies/studyCohorts.query.xml create mode 100644 SivStudies/resources/queries/studies/studyCohorts/.qview.xml diff --git a/SivStudies/resources/queries/studies/studies.query.xml b/SivStudies/resources/queries/studies/studies.query.xml new file mode 100644 index 00000000..bc903ae8 --- /dev/null +++ b/SivStudies/resources/queries/studies/studies.query.xml @@ -0,0 +1,21 @@ + + + + + + + true + false + false + Summary Information + + studies + studiesSummary + rowId + + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/studies/studies/.qview.xml b/SivStudies/resources/queries/studies/studies/.qview.xml new file mode 100644 index 00000000..69765c5a --- /dev/null +++ b/SivStudies/resources/queries/studies/studies/.qview.xml @@ -0,0 +1,18 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/studies/studiesSummary.query.xml b/SivStudies/resources/queries/studies/studiesSummary.query.xml new file mode 100644 index 00000000..6f01b8d1 --- /dev/null +++ b/SivStudies/resources/queries/studies/studiesSummary.query.xml @@ -0,0 +1,38 @@ + + + + + Studies Summary + + + + + + + + + + + + + + + # Animals + + + Sex(es) + + + SIV Challenges + + + ART Initiation(s) (DPI) + + + Interventions + + +
+ + + diff --git a/SivStudies/resources/queries/studies/studiesSummary.sql b/SivStudies/resources/queries/studies/studiesSummary.sql new file mode 100644 index 00000000..c79c61e2 --- /dev/null +++ b/SivStudies/resources/queries/studies/studiesSummary.sql @@ -0,0 +1,16 @@ +SELECT + s.rowId, + s.labelOrName, + s.description, + group_concat(DISTINCT a.cohortId.labelOrName, char(10)) as cohorts, + count(DISTINCT a.Id) as numAnimals, + group_concat(DISTINCT a.DataSets.Demographics.sex) as sexes, + group_concat(DISTINCT a.sivArt.allInfections, char(10)) as challenges, + group_concat(DISTINCT a.sivArt.artInitiationDPI, char(10)) as artInitiationsDPI, + group_concat(DISTINCT a.interventions.allInterventions, char(10)) as interventions, + + +FROM studies.studies s +LEFT JOIN study.assignment a ON (a.cohortId.studyId = s.rowId) + +GROUP BY s.rowId, s.labelOrName, s.description \ No newline at end of file diff --git a/SivStudies/resources/queries/studies/studyCohortSummary.query.xml b/SivStudies/resources/queries/studies/studyCohortSummary.query.xml new file mode 100644 index 00000000..8103e4aa --- /dev/null +++ b/SivStudies/resources/queries/studies/studyCohortSummary.query.xml @@ -0,0 +1,38 @@ + + + + + Study Cohort Summary + + + + + + Study Name + + + + + + Cohort Name + + + # Animals + + + Sex(es) + + + SIV Challenges + + + ART Initiation(s) (DPI) + + + Interventions + + +
+ + + diff --git a/SivStudies/resources/queries/studies/studyCohortSummary.sql b/SivStudies/resources/queries/studies/studyCohortSummary.sql new file mode 100644 index 00000000..f9626cf7 --- /dev/null +++ b/SivStudies/resources/queries/studies/studyCohortSummary.sql @@ -0,0 +1,16 @@ +SELECT + sc.rowId, + sc.studyId.labelOrName as studyName, + sc.labelOrName, + sc.studyId.description, + count(DISTINCT a.Id) as numAnimals, + group_concat(DISTINCT a.DataSets.Demographics.sex) as sexes, + group_concat(DISTINCT a.sivArt.allInfections, char(10)) as challenges, + group_concat(DISTINCT a.sivArt.artInitiationDPI, char(10)) as artInitiationsDPI, + group_concat(DISTINCT a.interventions.allInterventions, char(10)) as interventions + + +FROM studies.studyCohorts sc +LEFT JOIN study.assignment a ON (a.cohortId = sc.rowId) + +GROUP BY sc.rowId, sc.labelOrName, sc.studyId.description, sc.studyId.labelOrName \ No newline at end of file diff --git a/SivStudies/resources/queries/studies/studyCohorts.query.xml b/SivStudies/resources/queries/studies/studyCohorts.query.xml new file mode 100644 index 00000000..00428ae1 --- /dev/null +++ b/SivStudies/resources/queries/studies/studyCohorts.query.xml @@ -0,0 +1,21 @@ + + + + + + + true + false + false + Summary Information + + studies + studyCohortSummary + rowId + + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/studies/studyCohorts/.qview.xml b/SivStudies/resources/queries/studies/studyCohorts/.qview.xml new file mode 100644 index 00000000..2dc419f7 --- /dev/null +++ b/SivStudies/resources/queries/studies/studyCohorts/.qview.xml @@ -0,0 +1,27 @@ + \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java index 756f96ee..c604e013 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/PerformManualIdrStepsTask.java @@ -41,6 +41,9 @@ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJo updateVaccineInformation(pipelineJob); updateChallengeAnchorDates(pipelineJob); updateArtInitiationAnchorDates(pipelineJob); + updateJS46(); + + // TODO: Set other cohort-by-cohort params return new RecordedActionSet(); } @@ -283,4 +286,44 @@ public void setContainerUser(ContainerUser containerUser) { _containerUser = containerUser; } + + private void updateJS46() throws PipelineJobException + { + updateTreatmentRecords("JS46", new SimpleFilter(FieldKey.fromString("treatment"), "SIV - Unknown"), Map.of("treatment", "SIVmac239", "route", "IV")); + } + + private void updateTreatmentRecords(String cohortName, SimpleFilter treatmentFilter, final Map additionalProps) throws PipelineJobException + { + TableInfo assignments = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("assignment"); + List ids = new TableSelector(assignments, PageFlowUtil.set("Id"), new SimpleFilter(FieldKey.fromString("study"), cohortName), null).getArrayList(String.class); + treatmentFilter.addCondition(FieldKey.fromString("Id"), ids, CompareType.IN); + + TableInfo treatments = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study").getTable("treatments"); + List> toUpdate = new ArrayList<>(); + new TableSelector(treatments, PageFlowUtil.set("lsid"), treatmentFilter, null).forEachMap(rs -> { + Map toAdd = new CaseInsensitiveHashMap<>(rs); + toAdd.putAll(additionalProps); + toUpdate.add(toAdd); + }); + + if (!toUpdate.isEmpty()) + { + try + { + BatchValidationException bve = new BatchValidationException(); + + List> oldKeys = toUpdate.stream().map(x -> (Map)new CaseInsensitiveHashMap<>(Map.of("lsid", x.get("lsid")))).toList(); + treatments.getUpdateService().updateRows(_containerUser.getUser(), _containerUser.getContainer(), toUpdate, oldKeys, bve, null, null); + + if (bve.hasErrors()) + { + throw bve; + } + } + catch (SQLException | BatchValidationException | QueryUpdateServiceException | InvalidKeyException e) + { + throw new PipelineJobException(e); + } + } + } } From cb09daa42f65342950c28e0bb5d22f6c0d89d360 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 2 Dec 2025 17:39:40 -0800 Subject: [PATCH 14/14] Bump lz4-java version --- tcrdb/build.gradle | 7 +++++++ tcrdb/gradle.properties | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tcrdb/build.gradle b/tcrdb/build.gradle index 8218384e..b0eec461 100644 --- a/tcrdb/build.gradle +++ b/tcrdb/build.gradle @@ -5,6 +5,13 @@ repositories { mavenCentral() } +configurations.all { + resolutionStrategy { + // Related to: https://nvd.nist.gov/vuln/detail/CVE-2025-12183 + force "at.yawk.lz4:lz4-java:${lz4Version}" + } +} + dependencies { BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:singlecell", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiJarFile") diff --git a/tcrdb/gradle.properties b/tcrdb/gradle.properties index 50708a52..8b061f12 100644 --- a/tcrdb/gradle.properties +++ b/tcrdb/gradle.properties @@ -1 +1,3 @@ -repseqVersion=1.7.0 \ No newline at end of file +repseqVersion=1.7.0 +# Related to: https://nvd.nist.gov/vuln/detail/CVE-2025-12183 +lz4Version=1.9.0 \ No newline at end of file