From a842b3502eb3e010f6563dd7e4c98b2aaf7a53b9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 16 Jun 2025 09:29:53 -0700 Subject: [PATCH 01/52] Add calculated column for SIV Studies --- .../queries/study/assignment.query.xml | 4 +++ .../study/demographics/MHC Type.qview.xml | 1 + .../demographics/Project Summary.qview.xml | 18 ++++++++++++ .../study/demographicsProjects.query.xml | 28 +++++++++++++++++++ .../queries/study/demographicsProjects.sql | 10 +++++++ .../query/SivStudiesCustomizer.java | 7 +++++ 6 files changed, 68 insertions(+) create mode 100644 SivStudies/resources/queries/study/demographics/Project Summary.qview.xml create mode 100644 SivStudies/resources/queries/study/demographicsProjects.query.xml create mode 100644 SivStudies/resources/queries/study/demographicsProjects.sql diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml index 05d40e8c..1348a571 100644 --- a/SivStudies/resources/queries/study/assignment.query.xml +++ b/SivStudies/resources/queries/study/assignment.query.xml @@ -24,6 +24,10 @@ Category + + Description + true + diff --git a/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml b/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml index e0bd6d74..ed98208f 100644 --- a/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml +++ b/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml @@ -11,6 +11,7 @@ + diff --git a/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml b/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml new file mode 100644 index 00000000..30c79c60 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/Project Summary.qview.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsProjects.query.xml b/SivStudies/resources/queries/study/demographicsProjects.query.xml new file mode 100644 index 00000000..fc8f02e9 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsProjects.query.xml @@ -0,0 +1,28 @@ + + + + + Project Summary + + + true + true + + + Study Categories + + + All Studies + + + RhCMV Vaccines? + + + SIV/ART Projects? + + + categories +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsProjects.sql b/SivStudies/resources/queries/study/demographicsProjects.sql new file mode 100644 index 00000000..0a6666c7 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsProjects.sql @@ -0,0 +1,10 @@ +SELECT + s.Id, + count(s.Id) as totalTests, + group_concat(DISTINCT s.study, char(10)) as allStudies, + group_concat(DISTINCT s.category, char(10)) as categories, + + GROUP_CONCAT(distinct CASE WHEN s.category = 'RhCMV-Vaccines' THEN 'Yes' ELSE null END, char(10)) as rhCmvVaccines, + GROUP_CONCAT(distinct CASE WHEN s.category = 'SIV/ART' THEN 'Yes' ELSE null END, char(10)) as sivArt +FROM study.assignment s +GROUP BY s.Id \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index b1b4de83..cc78e74c 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -189,6 +189,13 @@ private void appendDemographicsColumns(AbstractTableInfo demographicsTable) colInfo.setLabel("MHC Genotypes"); demographicsTable.addColumn(colInfo); } + + if (demographicsTable.getColumn("projects") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(demographicsTable.getUserSchema(), "demographicsProjects", demographicsTable, "projects"); + colInfo.setLabel("Project Summary"); + demographicsTable.addColumn(colInfo); + } } private void appendPvlColumns(AbstractTableInfo ti, ColumnInfo subjectCol, ColumnInfo dateCol) From cf416dc0510077ec3e40b8b65b6a8e121f5bed33 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 16 Jun 2025 15:25:36 -0700 Subject: [PATCH 02/52] Update column names --- .../folderTypes/SIV Studies.folderType.xml | 34 +++++++++---------- .../queries/study/assignment.query.xml | 2 +- .../resources/queries/study/flags.query.xml | 2 +- .../queries/study/genetics.query.xml | 2 +- .../queries/study/immunizations.query.xml | 2 +- .../resources/queries/study/samples.query.xml | 2 +- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml index 2b6cda35..44129374 100644 --- a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml +++ b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml @@ -33,23 +33,23 @@ - - - - - - - - - - - - - - - - - + + dataBrowser + Data Browser + + + dataBrowser + + + + + + + Laboratory Data Browser + body + + + admin Admin diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml index 1348a571..8ce5db9f 100644 --- a/SivStudies/resources/queries/study/assignment.query.xml +++ b/SivStudies/resources/queries/study/assignment.query.xml @@ -5,7 +5,7 @@ - Date Added + Start Date Date diff --git a/SivStudies/resources/queries/study/flags.query.xml b/SivStudies/resources/queries/study/flags.query.xml index 6e2b23ed..d8795702 100644 --- a/SivStudies/resources/queries/study/flags.query.xml +++ b/SivStudies/resources/queries/study/flags.query.xml @@ -5,7 +5,7 @@ - Date Added + Start Date Date diff --git a/SivStudies/resources/queries/study/genetics.query.xml b/SivStudies/resources/queries/study/genetics.query.xml index 54b08dae..d29cc2b1 100644 --- a/SivStudies/resources/queries/study/genetics.query.xml +++ b/SivStudies/resources/queries/study/genetics.query.xml @@ -5,7 +5,7 @@ - Date Added + Date Date diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml index 833d0fde..06a2c2f7 100644 --- a/SivStudies/resources/queries/study/immunizations.query.xml +++ b/SivStudies/resources/queries/study/immunizations.query.xml @@ -5,7 +5,7 @@ - Date Added + Date Date diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml index 87ac5494..a0fef565 100644 --- a/SivStudies/resources/queries/study/samples.query.xml +++ b/SivStudies/resources/queries/study/samples.query.xml @@ -5,7 +5,7 @@ - Date Added + Date Date From a2ad03e78db46ed25e0c1afd9710cb73cbcb2906 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 10:51:35 -0700 Subject: [PATCH 03/52] SIV studies data cleanup --- SivStudies/resources/data/gender_codes.tsv | 2 +- SivStudies/resources/data/reports.tsv | 27 ++++++++++--------- SivStudies/resources/data/species.tsv | 2 +- SivStudies/resources/etls/siv-studies.xml | 22 +++++++++++++++ .../resources/queries/laboratory/reports.js | 11 ++++++++ .../queries/singlecell/sivDataSource.sql | 19 +++++++++++++ .../study/additionalDatatypes.query.xml | 22 +++++++++++++++ .../study/datasets/datasets_manifest.xml | 3 +++ .../study/datasets/datasets_metadata.xml | 23 ++++++++++++++++ 9 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 SivStudies/resources/queries/laboratory/reports.js create mode 100644 SivStudies/resources/queries/singlecell/sivDataSource.sql create mode 100644 SivStudies/resources/queries/study/additionalDatatypes.query.xml diff --git a/SivStudies/resources/data/gender_codes.tsv b/SivStudies/resources/data/gender_codes.tsv index 07b68195..783f8712 100644 --- a/SivStudies/resources/data/gender_codes.tsv +++ b/SivStudies/resources/data/gender_codes.tsv @@ -1,4 +1,4 @@ -v meaning origgender +value meaning origgender f Female f m Male m u Unknown \ No newline at end of file diff --git a/SivStudies/resources/data/reports.tsv b/SivStudies/resources/data/reports.tsv index 4dd48af5..8f493cd8 100644 --- a/SivStudies/resources/data/reports.tsv +++ b/SivStudies/resources/data/reports.tsv @@ -1,12 +1,15 @@ -reportname category reporttype reporttitle visible containerpath schemaname queryname viewname report datefieldname todayonly queryhaslocation sort_order QCStateLabelFieldName description -activeAssignments Assignments and Groups query Active Assignments true study Assignment Active Assignments date false false qcstate/publicdata This report shows the active assignments for each animal -assignmentHistory Assignments and Groups query Assignment History true study Assignment date false false qcstate/publicdata This report shows all assignments records for the animals -activeGroups Assignments and Groups query Active Groups true study animal_group_members Active Members date false false qcstate/publicdata This report shows the active assignments for each animal -groupHistory Assignments and Groups query Group History true study animal_group_members date false false qcstate/publicdata This report shows all assignments records for the animals -microbiology Lab Results query Microbiology true study Microbiology Results date false false qcstate/publicdata -biochemistry Lab Results js Biochemistry true study bloodChemistry date false false Contains results of chemistry panels. Can be displayed either by panel, or showing reference ranges -clinPathRuns Lab Results query Lab Runs true study Clinpath Runs date false false qcstate/publicdata Contains all clinpath requests -iStat Lab Results js iStat true study iStat date false false qcstate/publicdata Contains iStat results -hematology Lab Results js Hematology true study hematology date false false Contains hematology data showing cell subsets -parasitology Lab Results query Parasitology true study Parasitology Results date false false qcstate/publicdata Contains results of parasitology testing -urinalysis Lab Results js Urinalysis true study urinalysisResults date false false Contains urinalysis results +reporttype category schemaname queryname viewname reporttitle containerpath +query General study additionalDatatypes Additional Data +query General study weights Weights +query General study treatments Treatments/Medications +query General study labResults CBC/Chem +query General study assignment Project/Study Assignment +query General study geneticData Genetic Data / MHC +query General study procedures Procedures +query General study viralLoads Viral Loads +query General study samples Samples +query General study immunizations Immunizations +query General study flags Flags/Misc Information +query Demographics study demographics Demographics +query Demographics study demographics Project Summary Project Summary +query Demographics study demographics MHC Type MHC Typing diff --git a/SivStudies/resources/data/species.tsv b/SivStudies/resources/data/species.tsv index 0e063c4c..c071b727 100644 --- a/SivStudies/resources/data/species.tsv +++ b/SivStudies/resources/data/species.tsv @@ -1,4 +1,4 @@ -common scientific_name id_prefix mhc_prefix blood_per_kg max_draw_pct blood_draw_interval cites_code dateDisabled +common_name scientific_name id_prefix mhc_prefix blood_per_kg max_draw_pct blood_draw_interval cites_code dateDisabled Baboon 60 0.2 30 Cotton-top Tamarin Saguinus oedipus so Saoe 60 0.2 30 Cynomolgus Macaca fascicularis cy Mafa 60 0.2 30 diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml index f40e3ee2..41175547 100644 --- a/SivStudies/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -136,6 +136,28 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/SivStudies/resources/queries/laboratory/reports.js b/SivStudies/resources/queries/laboratory/reports.js new file mode 100644 index 00000000..c6f5589e --- /dev/null +++ b/SivStudies/resources/queries/laboratory/reports.js @@ -0,0 +1,11 @@ +function afterInsert() { + org.labkey.api.laboratory.LaboratoryService.get().clearDataProviderCache(); +} + +function afterUpdate() { + org.labkey.api.laboratory.LaboratoryService.get().clearDataProviderCache(); +} + +function afterDelete() { + org.labkey.api.laboratory.LaboratoryService.get().clearDataProviderCache(); +} \ No newline at end of file diff --git a/SivStudies/resources/queries/singlecell/sivDataSource.sql b/SivStudies/resources/queries/singlecell/sivDataSource.sql new file mode 100644 index 00000000..a19d960d --- /dev/null +++ b/SivStudies/resources/queries/singlecell/sivDataSource.sql @@ -0,0 +1,19 @@ +SELECT + +s1.subjectId as Id, +s1.date, + +('Assay(s): ' || assayType || char(10) || +'Tissue(s): ' || tissue || char(10) || +'Stims(s): ' || stims || char(10)) as description + +FROM (SELECT + s.subjectId, + s.sampleDate as date, + GROUP_CONCAT(DISTINCT s.tissue) as tissue, + GROUP_CONCAT(DISTINCT s.assayType) as assayType, + GROUP_CONCAT(DISTINCT s.stim) as stims + + FROM "/Labs/Bimber".singlecell.samples s + GROUP BY s.subjectId, s.sampleDate +) s1 \ No newline at end of file diff --git a/SivStudies/resources/queries/study/additionalDatatypes.query.xml b/SivStudies/resources/queries/study/additionalDatatypes.query.xml new file mode 100644 index 00000000..1fdf4468 --- /dev/null +++ b/SivStudies/resources/queries/study/additionalDatatypes.query.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + Category + + + Procedure + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml index 346bd030..5361e959 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml @@ -37,5 +37,8 @@ + + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index 24842c6c..0f0e3fe5 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -391,4 +391,27 @@ Procedures + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + varchar + + + varchar + + + Additional Datatypes +
From 409d39dbffe0ef69f15316bb6a2935cc80fa3f5a Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 11:30:26 -0700 Subject: [PATCH 04/52] Update TCRdb columns --- .../queries/tcrdb/clone_responses/.qview.xml | 22 +++++++++++++++++++ .../postgresql/tcrdb-15.54-15.55.sql | 7 ++++++ .../dbscripts/sqlserver/tcrdb-15.54-15.55.sql | 7 ++++++ tcrdb/resources/schemas/tcrdb.xml | 21 ++++++++++++++++++ tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 2 +- 5 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.54-15.55.sql create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.54-15.55.sql diff --git a/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml b/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml new file mode 100644 index 00000000..332f09b6 --- /dev/null +++ b/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.54-15.55.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.54-15.55.sql new file mode 100644 index 00000000..db2c1910 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.54-15.55.sql @@ -0,0 +1,7 @@ +ALTER TABLE tcrdb.clone_responses ADD vGene varchar(1000); +ALTER TABLE tcrdb.clone_responses ADD totalCells int; +ALTER TABLE tcrdb.clone_responses ADD totalCloneSize int; +ALTER TABLE tcrdb.clone_responses ADD fractionCloneActivated double precision; +ALTER TABLE tcrdb.clone_responses ADD totalCellsForSample int; +ALTER TABLE tcrdb.clone_responses ADD oddsRatio double precision; +ALTER TABLE tcrdb.clone_responses ADD enrichmentFDR double precision; diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.54-15.55.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.54-15.55.sql new file mode 100644 index 00000000..db2c1910 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.54-15.55.sql @@ -0,0 +1,7 @@ +ALTER TABLE tcrdb.clone_responses ADD vGene varchar(1000); +ALTER TABLE tcrdb.clone_responses ADD totalCells int; +ALTER TABLE tcrdb.clone_responses ADD totalCloneSize int; +ALTER TABLE tcrdb.clone_responses ADD fractionCloneActivated double precision; +ALTER TABLE tcrdb.clone_responses ADD totalCellsForSample int; +ALTER TABLE tcrdb.clone_responses ADD oddsRatio double precision; +ALTER TABLE tcrdb.clone_responses ADD enrichmentFDR double precision; diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index 88951dd5..bf64a4d9 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -134,6 +134,27 @@ Background Freq + + V-Gene + + + Total Cells + + + Total Clone Size + + + Fraction of Clone Activated + + + Total Cells For Sample + + + Enrichment (Odds Ratio) + + + Enrichment (FDR) + diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index dd7b9d91..6cd20382 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -46,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.54; + return 15.55; } @Override From 6f5d5fbfe2b202b41f32e9f0cdd963846a7d9f39 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 13:17:33 -0700 Subject: [PATCH 05/52] More cleanup in NavItems / TabbedReports --- SivStudies/resources/data/reports.tsv | 6 +++--- SivStudies/resources/folderTypes/SIV Studies.folderType.xml | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/SivStudies/resources/data/reports.tsv b/SivStudies/resources/data/reports.tsv index 8f493cd8..f5a43bfd 100644 --- a/SivStudies/resources/data/reports.tsv +++ b/SivStudies/resources/data/reports.tsv @@ -1,10 +1,10 @@ reporttype category schemaname queryname viewname reporttitle containerpath query General study additionalDatatypes Additional Data -query General study weights Weights +query General study weight Weights query General study treatments Treatments/Medications -query General study labResults CBC/Chem +query General study labwork CBC/Chem query General study assignment Project/Study Assignment -query General study geneticData Genetic Data / MHC +query General study genetics Genetic Data / MHC query General study procedures Procedures query General study viralLoads Viral Loads query General study samples Samples diff --git a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml index 44129374..0f25bbe0 100644 --- a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml +++ b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml @@ -64,6 +64,10 @@ SIV Studies Admin body + + Lab Tools + right + From 11a49da20af0e226cb45d1b8b283e92fe1b2b068 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 14:22:27 -0700 Subject: [PATCH 06/52] More granular cluster usage report --- .../notification/DiskUsageNotification.java | 118 +++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 7245af41..297d0dd5 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -1,10 +1,12 @@ package org.labkey.primeseq.notification; +import com.ibm.icu.util.Calendar; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.labkey.api.data.Container; import org.labkey.api.data.PropertyManager; import org.labkey.api.ldk.notification.Notification; @@ -16,10 +18,15 @@ import java.text.DateFormat; import java.text.NumberFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -115,7 +122,7 @@ private void getClusterUsage(Container c, User u, final StringBuilder msg) SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); String results = wrapper.executeWithOutput(Arrays.asList("ssh", "-q", "labkey_submit@arc", "sshare", "-U", "-u", "labkey_submit")); - msg.append("Cluster Usage:

"); + msg.append("Year-to-Date Cluster Usage:

"); msg.append(""); AtomicBoolean foundHeader = new AtomicBoolean(false); @@ -156,6 +163,21 @@ else if (!foundHeader.get()) msg.append("

\n"); + List> byMonth = getClusterUsageByMonth(Arrays.asList("bimberlab", "onprcgenetics"), 12); + byMonth.sort(Comparator.comparing(o -> String.valueOf(o.get("Account")))); + + msg.append("Cluster Usage By Month:

"); + msg.append("

AccountNormSharesRawUsageEffectiveUsageFairShare
"); + byMonth.forEach(map -> { + long cpu = (Long)map.get("CPU"); + long gpu = (Long)map.get("GPU"); + long units = (cpu/6000) + (gpu/600); + + msg.append(""); + }); + + msg.append("
AccountCPUGPUCompute Units
").append(map.get("Account")).append("").append(String.format("%,d", cpu)).append("").append(String.format("%,d", gpu)).append("").append(String.format("%,d", units)).append("
"); + msg.append("

\n"); } private void getDiskUsageStats(Container c, User u, final StringBuilder msg) @@ -196,4 +218,98 @@ private void getDiskUsageStats(Container c, User u, final StringBuilder msg) msg.append("

\n"); } + + private List> getClusterUsageByMonth(List accounts, int numMonths) + { + Calendar currentCal = Calendar.getInstance(); + int currentMonth = currentCal.get(Calendar.MONTH); + int currentYear = currentCal.get(Calendar.YEAR); + + List> results = new ArrayList<>(); + + int offset = 0; + while (offset < numMonths) + { + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.YEAR, currentYear); + cal.set(Calendar.MONTH, currentMonth - offset); + cal.set(Calendar.DAY_OF_MONTH, 1); + Date start = cal.getTime(); + + Calendar endCal = Calendar.getInstance(); + endCal.setTime(start); + endCal.add(Calendar.DATE, 1); + endCal.add(Calendar.MILLISECOND, -1); + Date end = endCal.getTime(); + + results.addAll(getClusterUsageForInterval(accounts, start, end)); + + offset++; + } + + return results; + } + + private @NotNull List> getClusterUsageForInterval(List accounts, Date start, Date end) + { + if (!SystemUtils.IS_OS_LINUX) + { + return Collections.emptyList(); + } + + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + + try + { + List args = new ArrayList<>(Arrays.asList("ssh", "-q", "labkey_submit@arc", "/usr/local/bin/sreport-accts-summary", "Accounts=" + StringUtils.join(accounts, ","))); + if (start != null) + { + args.add("Start=" + dateFormat.format(start)); + } + + if (end != null) + { + args.add("End=" + dateFormat.format(end)); + } + + SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); + String results = wrapper.executeWithOutput(args); + + AtomicBoolean foundHeader = new AtomicBoolean(false); + List> ret = Arrays.stream(results.split("\n")).map(x -> { + if (x.startsWith("Account|")) + { + foundHeader.set(true); + return null; + } + else if (!foundHeader.get()) + { + return null; + } + + String[] els = x.split("\\|"); + + if (els.length != 3) + { + _log.error("Unexpected line: " + StringUtils.join(els, "<>")); + return null; + } + + Map map = new HashMap<>(Map.of("Account", els[0], "CPU", Long.parseLong(els[1]), "GPU", Long.parseLong(els[2]))); + return map; + }).filter(Objects::nonNull).toList(); + + ret.forEach(map -> { + map.put("start", start); + map.put("end", end); + }); + + return ret; + } + catch (PipelineJobException e) + { + _log.error("Error fetching slurm summary", e); + return Collections.emptyList(); + } + } } \ No newline at end of file From d3166b52da09dcd211725a0c683cfb9a8a171489 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 14:34:18 -0700 Subject: [PATCH 07/52] Fix imports --- .../org/labkey/primeseq/notification/DiskUsageNotification.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 297d0dd5..99fe3183 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -1,6 +1,5 @@ package org.labkey.primeseq.notification; -import com.ibm.icu.util.Calendar; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.time.DurationFormatUtils; @@ -20,6 +19,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Collections; import java.util.Comparator; import java.util.Date; From cda4aa58630f32496aa72057f2467da6d6310056 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 15:59:29 -0700 Subject: [PATCH 08/52] Improve reports --- .../sivstudies/etl/SubjectScopedSelect.java | 91 +++++++++++++------ .../notification/DiskUsageNotification.java | 22 +++-- 2 files changed, 80 insertions(+), 33 deletions(-) diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index c2589a21..c7f1c754 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -14,6 +14,7 @@ import org.labkey.api.data.TableSelector; import org.labkey.api.di.DataIntegrationService; import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.CancelledException; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedActionSet; @@ -115,12 +116,20 @@ public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobExcepti List subjects = getSubjects(job.getLogger()); List> batches = Lists.partition(subjects, BATCH_SIZE); job.getLogger().info("Total batches: " + batches.size()); - batches.forEach(x -> processBatch(x, job.getLogger())); + batches.forEach(x -> processBatch(x, job.getLogger(), job)); return new RecordedActionSet(); } - private void processBatch(List subjects, Logger log) + private void checkCancelled(PipelineJob job) + { + if (job.isCancelled()) + { + throw new CancelledException(); + } + } + + private void processBatch(List subjects, Logger log, PipelineJob job) { log.info("processing batch with " + subjects.size() + " subjects"); TableInfo destinationTable = getDataDestinationTable(); @@ -146,11 +155,20 @@ private void processBatch(List subjects, Logger log) throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name())); } - Collection> existingRows = new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection(); + List> existingRows = new ArrayList<>(new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection()); if (!existingRows.isEmpty()) { - log.info("deleting " + existingRows.size() + " rows"); - qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), new ArrayList<>(existingRows), null, null); + List>> batches = Lists.partition(existingRows, 5000); + log.info("deleting " + existingRows.size() + " rows in " + batches.size() + " batches"); + int i = 0; + for (List> batch : batches) + { + i++; + log.info("batch " + i); + checkCancelled(job); + + qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), batch, null, null); + } } else { @@ -168,37 +186,58 @@ private void processBatch(List subjects, Logger log) { if (getMode() == MODE.TRUNCATE) { - log.info("inserting " + toImportOrUpdate.size() + " rows"); - BatchValidationException bve = new BatchValidationException(); - qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImportOrUpdate, bve, null, null); - if (bve.hasErrors()) + List>> batches = Lists.partition(toImportOrUpdate, 5000); + log.info("inserting " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches"); + + int i = 0; + for (List> batch : batches) { - throw bve; + i++; + log.info("batch " + i); + checkCancelled(job); + + BatchValidationException bve = new BatchValidationException(); + qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), batch, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } } } else if (getMode() == MODE.UPDATE_ONLY) { - log.info("updating " + toImportOrUpdate.size() + " rows"); - BatchValidationException bve = new BatchValidationException(); + List>> batches = Lists.partition(toImportOrUpdate, 5000); + log.info("updating " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches"); - Collection keyFields = destinationTable.getPkColumnNames(); - List> keys = toImportOrUpdate.stream().map(x -> { - Map map = new HashMap<>(); - for (String keyField : keyFields) - { - if (x.get(keyField) != null) + int i = 0; + for (List> batch : batches) + { + + i++; + log.info("batch " + i); + checkCancelled(job); + + BatchValidationException bve = new BatchValidationException(); + + Collection keyFields = destinationTable.getPkColumnNames(); + List> keys = batch.stream().map(x -> { + Map map = new HashMap<>(); + for (String keyField : keyFields) { - map.put(keyField, x.get(keyField)); + if (x.get(keyField) != null) + { + map.put(keyField, x.get(keyField)); + } } - } - return map; - }).toList(); + return map; + }).toList(); - qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), toImportOrUpdate, keys, bve, null, null); - if (bve.hasErrors()) - { - throw bve; + qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), batch, keys, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } } } else diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 99fe3183..8bacec8e 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -167,17 +167,18 @@ else if (!foundHeader.get()) byMonth.sort(Comparator.comparing(o -> String.valueOf(o.get("Account")))); msg.append("Cluster Usage By Month:

"); - msg.append(""); + msg.append("
AccountCPUGPUCompute Units
"); byMonth.forEach(map -> { long cpu = (Long)map.get("CPU"); long gpu = (Long)map.get("GPU"); - long units = (cpu/6000) + (gpu/600); + double units = (double)(cpu/6000) + (gpu/600); + Date start = (Date)map.get("Start"); - msg.append(""); + msg.append(""); }); msg.append("
AccountMonthCPUGPUCompute Units
").append(map.get("Account")).append("").append(String.format("%,d", cpu)).append("").append(String.format("%,d", gpu)).append("").append(String.format("%,d", units)).append("
").append(map.get("Account")).append("").append(getDateTimeFormat(c).format(start)).append("").append(String.format("%,d", cpu)).append("").append(String.format("%,d", gpu)).append("").append(String.format("%,d", units)).append("
"); - msg.append("

\n"); + msg.append("

\n"); } private void getDiskUsageStats(Container c, User u, final StringBuilder msg) @@ -216,7 +217,7 @@ private void getDiskUsageStats(Container c, User u, final StringBuilder msg) _log.error("Error running df", e); } - msg.append("

\n"); + msg.append("

\n"); } private List> getClusterUsageByMonth(List accounts, int numMonths) @@ -234,10 +235,17 @@ private List> getClusterUsageByMonth(List accounts, cal.set(Calendar.YEAR, currentYear); cal.set(Calendar.MONTH, currentMonth - offset); cal.set(Calendar.DAY_OF_MONTH, 1); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); Date start = cal.getTime(); Calendar endCal = Calendar.getInstance(); endCal.setTime(start); + endCal.set(Calendar.DAY_OF_MONTH, endCal.getActualMaximum(Calendar.DAY_OF_MONTH)); + + // Set to last second of the day: endCal.add(Calendar.DATE, 1); endCal.add(Calendar.MILLISECOND, -1); Date end = endCal.getTime(); @@ -300,8 +308,8 @@ else if (!foundHeader.get()) }).filter(Objects::nonNull).toList(); ret.forEach(map -> { - map.put("start", start); - map.put("end", end); + map.put("Start", start); + map.put("End", end); }); return ret; From 9b31cf462eefa8395415d6994c6ffde1f6361eae Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 16:23:59 -0700 Subject: [PATCH 09/52] Update cron string --- .../primeseq/notification/DiskUsageNotification.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 8bacec8e..57c145f8 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -15,7 +15,6 @@ import org.labkey.api.settings.LookAndFeelProperties; import java.text.DateFormat; -import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -36,8 +35,6 @@ public class DiskUsageNotification implements Notification { protected final static Logger _log = LogManager.getLogger(DiskUsageNotification.class); - private static final String lastSave = "lastSave"; - private NumberFormat _pctFormat = null; private static final String PROP_CATEGORY = "primeseq.DiskUsageNotification"; @@ -79,7 +76,7 @@ public DateFormat getDateTimeFormat(Container c) @Override public String getCronString() { - return "0 8 * * 1 ?"; + return "0 0 8 * * 1"; } @Override @@ -98,9 +95,6 @@ public String getMessageBodyHTML(Container c, User u) { Date start = new Date(); - _pctFormat = NumberFormat.getPercentInstance(); - _pctFormat.setMaximumFractionDigits(1); - StringBuilder msg = new StringBuilder(); getDiskUsageStats(c, u, msg); getClusterUsage(c, u, msg); From 3bfbe68600c0ef19961aec98fb5477bbe304f2e3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 16:31:07 -0700 Subject: [PATCH 10/52] Correct handling of double --- .../labkey/primeseq/notification/DiskUsageNotification.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 57c145f8..e7c4ae14 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -165,10 +165,10 @@ else if (!foundHeader.get()) byMonth.forEach(map -> { long cpu = (Long)map.get("CPU"); long gpu = (Long)map.get("GPU"); - double units = (double)(cpu/6000) + (gpu/600); + double units = ((double)cpu/6000) + ((double)gpu/600); Date start = (Date)map.get("Start"); - msg.append("").append(map.get("Account")).append("").append(getDateTimeFormat(c).format(start)).append("").append(String.format("%,d", cpu)).append("").append(String.format("%,d", gpu)).append("").append(String.format("%,d", units)).append(""); + msg.append("").append(map.get("Account")).append("").append(getDateTimeFormat(c).format(start)).append("").append(String.format("%,d", cpu)).append("").append(String.format("%,d", gpu)).append("").append(String.format("%,.2f", units)).append(""); }); msg.append(""); From 3930d84f447cfc89d8d8c6dbe5004ed8f323eede Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 17:22:05 -0700 Subject: [PATCH 11/52] Fix cron string --- .../org/labkey/primeseq/notification/DiskUsageNotification.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index e7c4ae14..473cb132 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -76,7 +76,7 @@ public DateFormat getDateTimeFormat(Container c) @Override public String getCronString() { - return "0 0 8 * * 1"; + return "0 8 * * 1 ?"; } @Override From 7eabc589d2ebe001f6334de577bc542732b384cc Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Jun 2025 17:54:03 -0700 Subject: [PATCH 12/52] Improve notification formatting --- .../notification/DiskUsageNotification.java | 47 ++++++++----------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 473cb132..799965cf 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -7,7 +7,6 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.labkey.api.data.Container; -import org.labkey.api.data.PropertyManager; import org.labkey.api.ldk.notification.Notification; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.security.User; @@ -36,8 +35,6 @@ public class DiskUsageNotification implements Notification { protected final static Logger _log = LogManager.getLogger(DiskUsageNotification.class); - private static final String PROP_CATEGORY = "primeseq.DiskUsageNotification"; - @Override public String getName() { @@ -85,11 +82,6 @@ public String getScheduleDescription() return "Every Monday at 8AM"; } - private Map getSavedValues(Container c) - { - return PropertyManager.getProperties(c, PROP_CATEGORY); - } - @Override public String getMessageBodyHTML(Container c, User u) { @@ -111,12 +103,29 @@ private void getClusterUsage(Container c, User u, final StringBuilder msg) return; } + List> byMonth = getClusterUsageByMonth(Arrays.asList("bimberlab", "onprcgenetics"), 12); + byMonth.sort(Comparator.comparing(o -> String.valueOf(o.get("Account")))); + + msg.append("Cluster Usage By Month:

"); + msg.append(""); + byMonth.forEach(map -> { + long cpu = (Long)map.get("CPU"); + long gpu = (Long)map.get("GPU"); + double units = ((double)cpu/6000) + ((double)gpu/600); + Date start = (Date)map.get("Start"); + + msg.append(""); + }); + + msg.append("
AccountMonthCPUGPUCompute Units
").append(map.get("Account")).append("").append(getDateTimeFormat(c).format(start)).append("").append(String.format("%,d", cpu)).append("").append(String.format("%,d", gpu)).append("").append(String.format("%,.2f", units)).append("
"); + msg.append("

\n"); + try { SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); String results = wrapper.executeWithOutput(Arrays.asList("ssh", "-q", "labkey_submit@arc", "sshare", "-U", "-u", "labkey_submit")); - msg.append("Year-to-Date Cluster Usage:

"); + msg.append("Cluster Priority By Account:

"); msg.append(""); AtomicBoolean foundHeader = new AtomicBoolean(false); @@ -149,30 +158,12 @@ else if (!foundHeader.get()) msg.append(""); }); msg.append("
AccountNormSharesRawUsageEffectiveUsageFairShare
"); + msg.append("

\n"); } catch (PipelineJobException e) { _log.error("Error fetching slurm summary", e); } - - msg.append("

\n"); - - List> byMonth = getClusterUsageByMonth(Arrays.asList("bimberlab", "onprcgenetics"), 12); - byMonth.sort(Comparator.comparing(o -> String.valueOf(o.get("Account")))); - - msg.append("Cluster Usage By Month:

"); - msg.append(""); - byMonth.forEach(map -> { - long cpu = (Long)map.get("CPU"); - long gpu = (Long)map.get("GPU"); - double units = ((double)cpu/6000) + ((double)gpu/600); - Date start = (Date)map.get("Start"); - - msg.append(""); - }); - - msg.append("
AccountMonthCPUGPUCompute Units
").append(map.get("Account")).append("").append(getDateTimeFormat(c).format(start)).append("").append(String.format("%,d", cpu)).append("").append(String.format("%,d", gpu)).append("").append(String.format("%,.2f", units)).append("
"); - msg.append("

\n"); } private void getDiskUsageStats(Container c, User u, final StringBuilder msg) From 33b6c42b7cf51c59451d4e0ec99e61a0412428a0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 19 Jun 2025 07:08:21 -0700 Subject: [PATCH 13/52] Fix containerPath in ETL --- SivStudies/resources/etls/siv-studies.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml index 41175547..6bb6fc2d 100644 --- a/SivStudies/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -144,7 +144,7 @@ - + From efd532db742bfe312c51ebf9094d80787b6caf06 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 19 Jun 2025 17:49:03 -0700 Subject: [PATCH 14/52] Additional SIV queries --- SivStudies/resources/data/labwork_types.tsv | 61 ++++++++++++++++++ SivStudies/resources/data/lookup_sets.tsv | 3 +- SivStudies/resources/data/reports.tsv | 23 +++---- .../queries/singlecell/sivDataSource.sql | 6 +- .../queries/study/chemistryPivot.query.xml | 19 ++++++ .../queries/study/chemistryPivot.sql | 24 +++++++ .../queries/study/hematologyPivot.query.xml | 19 ++++++ .../queries/study/hematologyPivot.sql | 24 +++++++ .../resources/queries/study/labwork.query.xml | 6 ++ .../queries/study/viralloads.query.xml | 2 +- .../query/SivStudiesCustomizer.java | 62 ++++++++++++++----- .../postgresql/tcrdb-15.55-15.56.sql | 1 + .../dbscripts/sqlserver/tcrdb-15.55-15.56.sql | 1 + tcrdb/resources/schemas/tcrdb.xml | 3 + tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 2 +- 15 files changed, 222 insertions(+), 34 deletions(-) create mode 100644 SivStudies/resources/data/labwork_types.tsv create mode 100644 SivStudies/resources/queries/study/chemistryPivot.query.xml create mode 100644 SivStudies/resources/queries/study/chemistryPivot.sql create mode 100644 SivStudies/resources/queries/study/hematologyPivot.query.xml create mode 100644 SivStudies/resources/queries/study/hematologyPivot.sql create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.55-15.56.sql create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.55-15.56.sql diff --git a/SivStudies/resources/data/labwork_types.tsv b/SivStudies/resources/data/labwork_types.tsv new file mode 100644 index 00000000..20a06517 --- /dev/null +++ b/SivStudies/resources/data/labwork_types.tsv @@ -0,0 +1,61 @@ +value title units sort_order category +GLUC Glucose mg/dL 1 Chemistry +BUN Blood urea nitrogen mg/dL 2 Chemistry +CREAT Creatinine mg/dL 3 Chemistry +CPK Creatine phosphokinase U/L 4 Chemistry +CHOL Cholesterol mg/dl 5 Chemistry +TRIG Triglyceride mg/dL 6 Chemistry +SGOT Serum glutamic oxaloacetic transaminase IU/L 7 Chemistry +LDL Low-Density Lipoprotein mg/dL 8 Chemistry +LDH Lactate dehydrogenase IU/L 9 Chemistry +TB Total Bilirubin mg/dL 10 Chemistry +GGT Gamma-glutamyltransferase IU/L 11 Chemistry +SGPT Serum glutamic pyruvic transaminase IU/L 12 Chemistry +TP Total Protein g/dL 13 Chemistry +ALB Albumin g/dL 14 Chemistry +ALKP Alkaline Phosphatase IU/L 15 Chemistry +CA Calcium mg/dL 16 Chemistry +PHOS Phosphorus mg/dL 17 Chemistry +FE Iron ?g/dL 18 Chemistry +K Potassium mmol/L 20 Chemistry +UA Uric Acid mg/dL 22 Chemistry +A/P Albumin/Protein Ratio ratio 999 Chemistry +B/C BUN/Creatinine Ratio ratio 999 Chemistry +CL Chloride mEq/L 999 Chemistry +FIBR Fibrinogen mg/dL 999 Chemistry +HDL High-Density Lipoprotein mg/dL 999 Chemistry +WBC White Blood Cells 10^3/uL 1 Hematology +NEUT% Neutrophils % 2 Hematology +BAND% Bands 3 Hematology +LYMPH% Lymphocytes % 4 Hematology +MONO% Monocytes % 5 Hematology +EOS% Eosinophils % 6 Hematology +BASO% Basophils % 7 Hematology +HCT Hematocrit % 8 Hematology +HGB Hemoglobin 9 Hematology +RBC Red Blood Cells 10^6/uL 10 Hematology +MCV Mean corpuscular volume fL 11 Hematology +MCH Mean corpuscular hemoglobin picograms 12 Hematology +MCHC Mean corpuscular hemoglobin concentration g/dL 13 Hematology +RETIC Reticulocytes % 14 Hematology +PLT Platelets 10^3/uL 15 Hematology +MYELO Myelocytes % 999 Hematology +MPV Mean platelet volume fl 999 Hematology +Anisocytosis Anisocytosis 999 Hematology +GLOB Globulin g/dL 999 Hematology +Hypochromic RBC Hypochromic RBC 999 Hematology +LUC Large unstained cells 999 Hematology +Macrocytic RBC Macrocytic RBC 999 Hematology +METAMYELO Metamyelocytes % 999 Hematology +Microcytic RBC Microcytic RBC 999 Hematology +NRBC Nucleated red blood cells 999 Hematology +PCV Packed cell volume % 999 Hematology +Polychromasia RBC Polychromasia RBC 999 Hematology +MPMN Neutrophils 999 Hematology +SEDRate Sedimentation rate 999 Hematology +RDW % 999 Hematology +NEUT# Neutrophil # 10^3/uL 16 Hematology +LYMPH# Lymphocyte # 10^3/uL 17 Hematology +MONO# Monocyte # 10^3/uL 18 Hematology +EOS# Eosinophil # 10^3/uL 19 Hematology +BASO# Basophil # 10^3/uL 20 Hematology diff --git a/SivStudies/resources/data/lookup_sets.tsv b/SivStudies/resources/data/lookup_sets.tsv index 8e2b9482..445f8e0b 100644 --- a/SivStudies/resources/data/lookup_sets.tsv +++ b/SivStudies/resources/data/lookup_sets.tsv @@ -6,4 +6,5 @@ dosage_units Dosage Units unit gender_codes Gender Codes geographic_origins Geographic Origins origin routes Routes route -volume_units Volume Units unit \ No newline at end of file +volume_units Volume Units unit +labwork_types Lab Tests test title \ No newline at end of file diff --git a/SivStudies/resources/data/reports.tsv b/SivStudies/resources/data/reports.tsv index f5a43bfd..bae6925e 100644 --- a/SivStudies/resources/data/reports.tsv +++ b/SivStudies/resources/data/reports.tsv @@ -1,15 +1,16 @@ reporttype category schemaname queryname viewname reporttitle containerpath query General study additionalDatatypes Additional Data -query General study weight Weights -query General study treatments Treatments/Medications -query General study labwork CBC/Chem -query General study assignment Project/Study Assignment -query General study genetics Genetic Data / MHC -query General study procedures Procedures -query General study viralLoads Viral Loads +query Clinical study weight Weights +query Clinical study treatments Treatments/Medications +query Clinical study chemistryPivot Blood Chemistry +query Clinical study hematologyPivot CBC/Hematology +query Research study assignment Project/Study Assignment +query Research study genetics Genetic Data / MHC +query Clinical study procedures Procedures +query Research study viralLoads Viral Loads query General study samples Samples -query General study immunizations Immunizations +query Research study immunizations Immunizations query General study flags Flags/Misc Information -query Demographics study demographics Demographics -query Demographics study demographics Project Summary Project Summary -query Demographics study demographics MHC Type MHC Typing +query General study demographics Demographics +query General study demographics Project Summary Project Summary +query General study demographics MHC Type MHC Typing diff --git a/SivStudies/resources/queries/singlecell/sivDataSource.sql b/SivStudies/resources/queries/singlecell/sivDataSource.sql index a19d960d..1cf6eab1 100644 --- a/SivStudies/resources/queries/singlecell/sivDataSource.sql +++ b/SivStudies/resources/queries/singlecell/sivDataSource.sql @@ -10,9 +10,9 @@ s1.date, FROM (SELECT s.subjectId, s.sampleDate as date, - GROUP_CONCAT(DISTINCT s.tissue) as tissue, - GROUP_CONCAT(DISTINCT s.assayType) as assayType, - GROUP_CONCAT(DISTINCT s.stim) as stims + GROUP_CONCAT(DISTINCT s.tissue, ', ') as tissue, + GROUP_CONCAT(DISTINCT s.assayType, ', ') as assayType, + GROUP_CONCAT(DISTINCT s.stim, ', ') as stims FROM "/Labs/Bimber".singlecell.samples s GROUP BY s.subjectId, s.sampleDate diff --git a/SivStudies/resources/queries/study/chemistryPivot.query.xml b/SivStudies/resources/queries/study/chemistryPivot.query.xml new file mode 100644 index 00000000..4ff5a950 --- /dev/null +++ b/SivStudies/resources/queries/study/chemistryPivot.query.xml @@ -0,0 +1,19 @@ + + + + + Chemistry Results + + + + + study + animal + Id + + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/chemistryPivot.sql b/SivStudies/resources/queries/study/chemistryPivot.sql new file mode 100644 index 00000000..139f2cb9 --- /dev/null +++ b/SivStudies/resources/queries/study/chemistryPivot.sql @@ -0,0 +1,24 @@ +SELECT + b.Id, + b.date, + b.method, + b.test, + group_concat(b.result) as result + +FROM (SELECT + b.Id, + b.date, + b.test, + b.method, + CASE + WHEN b.result IS NULL THEN b.qualresult + ELSE CAST(CAST(b.result AS float) AS VARCHAR) + END as result + FROM study.labwork b + WHERE b.test.category = 'Chemistry' AND b.test.sort_order != 999 +) b + +GROUP BY b.id, b.date, b.test, b.method +PIVOT result BY test IN (select value from studies.labwork_types t WHERE t.sort_order != 999 AND t.category = 'Chemistry' order by sort_order) + + diff --git a/SivStudies/resources/queries/study/hematologyPivot.query.xml b/SivStudies/resources/queries/study/hematologyPivot.query.xml new file mode 100644 index 00000000..6ae5b45c --- /dev/null +++ b/SivStudies/resources/queries/study/hematologyPivot.query.xml @@ -0,0 +1,19 @@ + + + + + Hematology Results + + + + + study + animal + Id + + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/hematologyPivot.sql b/SivStudies/resources/queries/study/hematologyPivot.sql new file mode 100644 index 00000000..c15d5996 --- /dev/null +++ b/SivStudies/resources/queries/study/hematologyPivot.sql @@ -0,0 +1,24 @@ +SELECT + b.Id, + b.date, + b.method, + b.test, + group_concat(b.result) as result + +FROM (SELECT + b.Id, + b.date, + b.test, + b.method, + CASE + WHEN b.result IS NULL THEN b.qualresult + ELSE CAST(CAST(b.result AS float) AS VARCHAR) + END as result + FROM study.labwork b + WHERE b.test.category = 'Hematology' AND b.test.sort_order != 999 +) b + +GROUP BY b.id, b.date, b.test, b.method +PIVOT result BY test IN (select value from studies.labwork_types t WHERE t.sort_order != 999 AND t.category = 'Hematology' order by sort_order) + + diff --git a/SivStudies/resources/queries/study/labwork.query.xml b/SivStudies/resources/queries/study/labwork.query.xml index 99e85a14..7f411731 100644 --- a/SivStudies/resources/queries/study/labwork.query.xml +++ b/SivStudies/resources/queries/study/labwork.query.xml @@ -13,6 +13,12 @@ Test + + studies + labwork_types + value + + Result diff --git a/SivStudies/resources/queries/study/viralloads.query.xml b/SivStudies/resources/queries/study/viralloads.query.xml index ef00f0e2..4ac01867 100644 --- a/SivStudies/resources/queries/study/viralloads.query.xml +++ b/SivStudies/resources/queries/study/viralloads.query.xml @@ -15,7 +15,7 @@ Assay Type - target + Target LOD diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index cc78e74c..f89cbf02 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -5,9 +5,12 @@ import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.SQLFragment; import org.labkey.api.data.TableInfo; import org.labkey.api.data.WrappedColumn; import org.labkey.api.ldk.table.AbstractTableCustomizer; +import org.labkey.api.query.ExprColumn; import org.labkey.api.query.LookupForeignKey; import org.labkey.api.query.QueryDefinition; import org.labkey.api.query.QueryException; @@ -15,6 +18,7 @@ import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; import org.labkey.api.security.User; +import org.labkey.api.study.Dataset; import org.labkey.api.study.DatasetTable; import org.labkey.api.util.logging.LogHelper; @@ -23,9 +27,11 @@ public class SivStudiesCustomizer extends AbstractTableCustomizer { - public static final String ID_COL = "Id"; private static final Logger _log = LogHelper.getLogger(SivStudiesCustomizer.class, "Table customization for the SIV Studies module"); + public static final String ID_COL = "Id"; + public static final String DATE_COL = "Date"; + @Override public void customize(TableInfo tableInfo) { @@ -43,7 +49,8 @@ public void performDatasetCustomization(DatasetTable ds) if (!ds.getDataset().isDemographicData()) { - appendAgeAtTimeCol(ds.getUserSchema(), ati, "date"); + appendAgeAtTimeCol(ds.getUserSchema(), ati, DATE_COL); + appendPvlColumns(ds, ID_COL, DATE_COL); } if ("demographics".equalsIgnoreCase(ds.getName())) @@ -196,24 +203,45 @@ private void appendDemographicsColumns(AbstractTableInfo demographicsTable) colInfo.setLabel("Project Summary"); demographicsTable.addColumn(colInfo); } + + if (demographicsTable.getColumn("immunizations") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(demographicsTable.getUserSchema(), "demographicsImmunizations", demographicsTable, "immunizations"); + colInfo.setLabel("Immunization Summary"); + demographicsTable.addColumn(colInfo); + } + + if (demographicsTable.getColumn("challenges") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(demographicsTable.getUserSchema(), "demographicsChallenges", demographicsTable, "challenges"); + colInfo.setLabel("Challenge Summary"); + demographicsTable.addColumn(colInfo); + } } - private void appendPvlColumns(AbstractTableInfo ti, ColumnInfo subjectCol, ColumnInfo dateCol) + private void appendPvlColumns(DatasetTable ds, String subjectCol, String dateCol) { - Container target = ti.getUserSchema().getContainer().isWorkbookOrTab() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer(); - -// TableInfo data = schema.createDataTable(null); -// String tableName = data.getDomain().getStorageTableName(); -// -// String name = "viralLoad"; -// if (ti.getColumn(name) == null) -// { -// SQLFragment sql = new SQLFragment("(SELECT avg(viralLoad) as expr FROM assayresult." + tableName + " t WHERE t.subjectId = " + ExprColumn.STR_TABLE_ALIAS + "." + subjectCol.getName() + " AND CAST(t.date AS DATE) = CAST(" + ExprColumn.STR_TABLE_ALIAS + "." + dateCol.getName() + " AS DATE))"); -// ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.DOUBLE, subjectCol, dateCol); -// newCol.setDescription("Displays the viral load from this timepoint, if present"); -// newCol.setLabel("Viral Load (copies/mL)"); -// ti.addColumn(newCol); -// } + final String name = "viralLoad"; + if (ds.getColumn(name) != null) + { + return; + } + + Dataset vl = ds.getDataset().getStudy().getDatasetByName("viralloads"); + if (vl == null) + { + return; + } + + if (ds instanceof AbstractTableInfo ti) + { + final String tableName = vl.getDomain().getStorageTableName(); + SQLFragment sql = new SQLFragment("(SELECT CASE WHEN count(t.*) == 1 THEN max(t.viralLoad) ELSE null END as expr FROM studydataset." + tableName + " t WHERE t." + subjectCol + " = " + ExprColumn.STR_TABLE_ALIAS + "." + subjectCol + " AND CAST(t.date AS DATE) = CAST(" + ExprColumn.STR_TABLE_ALIAS + "." + dateCol + " AS DATE) AND t.category = 'Plasma' AND t.target = 'SIV')"); + ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.DOUBLE, ti.getColumn(subjectCol), ti.getColumn(dateCol)); + newCol.setDescription("Displays the viral load from this timepoint, if present"); + newCol.setLabel("SIV PVL (copies/mL)"); + ti.addColumn(newCol); + } } private BaseColumnInfo getWrappedIdCol(UserSchema targetQueryUserSchema, String targetQueryName, AbstractTableInfo demographicsTable, String colName) diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.55-15.56.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.55-15.56.sql new file mode 100644 index 00000000..b42680c3 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.55-15.56.sql @@ -0,0 +1 @@ +ALTER TABLE tcrdb.clone_responses ADD jGene varchar(1000); diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.55-15.56.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.55-15.56.sql new file mode 100644 index 00000000..b42680c3 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.55-15.56.sql @@ -0,0 +1 @@ +ALTER TABLE tcrdb.clone_responses ADD jGene varchar(1000); diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index bf64a4d9..57d407dd 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -137,6 +137,9 @@ V-Gene + + J-Gene + Total Cells diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 6cd20382..a2512dd6 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -46,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.55; + return 15.56; } @Override From 6d19defc5fef8df65d72f21fb6a09889f6ea27bf Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 19 Jun 2025 18:15:51 -0700 Subject: [PATCH 15/52] Additional SIV Lookup data --- SivStudies/resources/data/lookup_sets.tsv | 3 ++- SivStudies/resources/data/vl_sample_types.tsv | 3 +++ .../queries/study/demographics/MHC Type.qview.xml | 1 + SivStudies/resources/queries/study/viralloads.query.xml | 6 ++++++ .../labkey/sivstudies/query/SivStudiesCustomizer.java | 9 ++++++--- 5 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 SivStudies/resources/data/vl_sample_types.tsv diff --git a/SivStudies/resources/data/lookup_sets.tsv b/SivStudies/resources/data/lookup_sets.tsv index 445f8e0b..d831d31b 100644 --- a/SivStudies/resources/data/lookup_sets.tsv +++ b/SivStudies/resources/data/lookup_sets.tsv @@ -7,4 +7,5 @@ gender_codes Gender Codes geographic_origins Geographic Origins origin routes Routes route volume_units Volume Units unit -labwork_types Lab Tests test title \ No newline at end of file +labwork_types Lab Tests test title +vl_sample_types Lab Tests test \ No newline at end of file diff --git a/SivStudies/resources/data/vl_sample_types.tsv b/SivStudies/resources/data/vl_sample_types.tsv new file mode 100644 index 00000000..51943b0b --- /dev/null +++ b/SivStudies/resources/data/vl_sample_types.tsv @@ -0,0 +1,3 @@ +value +Plasma +Cells \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml b/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml index ed98208f..063bf9c6 100644 --- a/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml +++ b/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml @@ -7,6 +7,7 @@ + diff --git a/SivStudies/resources/queries/study/viralloads.query.xml b/SivStudies/resources/queries/study/viralloads.query.xml index 4ac01867..dcef9ed6 100644 --- a/SivStudies/resources/queries/study/viralloads.query.xml +++ b/SivStudies/resources/queries/study/viralloads.query.xml @@ -10,6 +10,12 @@ Sample Type + + studies + vl_sample_types + value + + Assay Type diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index f89cbf02..e9133949 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -219,7 +219,7 @@ private void appendDemographicsColumns(AbstractTableInfo demographicsTable) } } - private void appendPvlColumns(DatasetTable ds, String subjectCol, String dateCol) + private void appendPvlColumns(DatasetTable ds, String subjectColName, String dateColName) { final String name = "viralLoad"; if (ds.getColumn(name) != null) @@ -235,9 +235,12 @@ private void appendPvlColumns(DatasetTable ds, String subjectCol, String dateCol if (ds instanceof AbstractTableInfo ti) { + ColumnInfo subjectCol = ti.getColumn(subjectColName); + ColumnInfo dateCol = ti.getColumn(dateColName); + final String tableName = vl.getDomain().getStorageTableName(); - SQLFragment sql = new SQLFragment("(SELECT CASE WHEN count(t.*) == 1 THEN max(t.viralLoad) ELSE null END as expr FROM studydataset." + tableName + " t WHERE t." + subjectCol + " = " + ExprColumn.STR_TABLE_ALIAS + "." + subjectCol + " AND CAST(t.date AS DATE) = CAST(" + ExprColumn.STR_TABLE_ALIAS + "." + dateCol + " AS DATE) AND t.category = 'Plasma' AND t.target = 'SIV')"); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.DOUBLE, ti.getColumn(subjectCol), ti.getColumn(dateCol)); + SQLFragment sql = new SQLFragment("(SELECT CASE WHEN count(t.result) = 1 THEN max(t.result) ELSE null END as expr FROM studydataset." + tableName + " t WHERE t.participantid = " + ExprColumn.STR_TABLE_ALIAS + ".participantid AND CAST(t.date AS DATE) = CAST(" + ExprColumn.STR_TABLE_ALIAS + ".date AS DATE) AND t.sampletype = 'Plasma' AND t.target = 'SIV')"); + ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.DOUBLE, subjectCol, dateCol); newCol.setDescription("Displays the viral load from this timepoint, if present"); newCol.setLabel("SIV PVL (copies/mL)"); ti.addColumn(newCol); From 2de2d68c6a038b0590f598870b8d10045356a718 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 Jun 2025 09:11:52 -0700 Subject: [PATCH 16/52] Initialize schema to manage studies/cohorts/timepoints --- .../labkey/sivstudies/SivStudiesModule.java | 6 +- .../study/ArtInitiationEventProvider.java | 58 +++++++++++++++++ .../study/SivInfectionEventProvider.java | 62 +++++++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 SivStudies/src/org/labkey/sivstudies/study/ArtInitiationEventProvider.java create mode 100644 SivStudies/src/org/labkey/sivstudies/study/SivInfectionEventProvider.java diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java index 020e9889..4bedeb33 100644 --- a/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java @@ -21,6 +21,9 @@ import org.labkey.api.data.Container; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.module.ModuleContext; +import org.labkey.api.studies.StudiesService; +import org.labkey.sivstudies.study.ArtInitiationEventProvider; +import org.labkey.sivstudies.study.SivInfectionEventProvider; import java.util.Collection; import java.util.Collections; @@ -51,7 +54,8 @@ protected void init() @Override public void doStartupAfterSpringConfig(ModuleContext moduleContext) { - + StudiesService.get().registerEventProvider(new SivInfectionEventProvider()); + StudiesService.get().registerEventProvider(new ArtInitiationEventProvider()); } @Override diff --git a/SivStudies/src/org/labkey/sivstudies/study/ArtInitiationEventProvider.java b/SivStudies/src/org/labkey/sivstudies/study/ArtInitiationEventProvider.java new file mode 100644 index 00000000..c627d406 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/study/ArtInitiationEventProvider.java @@ -0,0 +1,58 @@ +package org.labkey.sivstudies.study; + +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.User; +import org.labkey.api.studies.study.AbstractEventProvider; +import org.labkey.api.study.DatasetTable; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.sivstudies.SivStudiesModule; + +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +public class ArtInitiationEventProvider extends AbstractEventProvider +{ + public ArtInitiationEventProvider() + { + super("ART_Initiation", "ART Initiation", "This is the first date when the subject was assigned to the study, as defined in the study assignment table", ModuleLoader.getInstance().getModule(SivStudiesModule.class)); + } + + @Override + protected Map inferDatesRaw(Collection subjectList, Container c, User u) + { + TableInfo ti = getTable(c, u, "study", "treatments"); + if (ti == null) + { + return null; + } + + if (ti instanceof DatasetTable ds) + { + Map ret = new HashMap<>(); + final String subjectCol = ds.getDataset().getStudy().getSubjectColumnName(); + new TableSelector(ti, PageFlowUtil.set(subjectCol, "date"), new SimpleFilter(FieldKey.fromString(subjectCol), subjectList, CompareType.IN), null).forEachResults(rs -> { + String subjectId = rs.getString(FieldKey.fromString(subjectCol)); + Date date = rs.getDate(FieldKey.fromString("date")); + + if (!ret.containsKey(subjectId) || date.before(ret.get(subjectId))) + { + ret.put(subjectId, date); + } + }); + + return ret; + } + else + { + throw new IllegalStateException("Expected study.assignment to be a DatasetTable"); + } + } +} diff --git a/SivStudies/src/org/labkey/sivstudies/study/SivInfectionEventProvider.java b/SivStudies/src/org/labkey/sivstudies/study/SivInfectionEventProvider.java new file mode 100644 index 00000000..6ff2c177 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/study/SivInfectionEventProvider.java @@ -0,0 +1,62 @@ +package org.labkey.sivstudies.study; + +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.studies.study.AbstractEventProvider; +import org.labkey.api.study.DatasetTable; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.sivstudies.SivStudiesModule; + +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +public class SivInfectionEventProvider extends AbstractEventProvider +{ + public SivInfectionEventProvider() + { + super("SIV_Infection", "SIV Infection", "This is the official date of SIV infection, used to calculate days-post-infection", ModuleLoader.getInstance().getModule(SivStudiesModule.class)); + } + + @Override + protected Map inferDatesRaw(Collection subjectList, Container c, User u) + { + UserSchema us = QueryService.get().getUserSchema(u, c, "study"); + if (us == null) + { + return Collections.emptyMap(); + } + + TableInfo ti = us.getTable("assignment"); + if (ti == null || !ti.hasPermission(u, ReadPermission.class)) + { + return Collections.emptyMap(); + } + + if (ti instanceof DatasetTable ds) + { + Map ret = new HashMap<>(); + final String subjectCol = ds.getDataset().getStudy().getSubjectColumnName(); + new TableSelector(ti, PageFlowUtil.set(subjectCol, "date"), new SimpleFilter(FieldKey.fromString(subjectCol), subjectList, CompareType.IN), null).forEachResults(rs -> { + ret.put(rs.getString(FieldKey.fromString(subjectCol)), rs.getDate(FieldKey.fromString("date"))); + }); + + return ret; + } + else + { + throw new IllegalStateException("Expected study.assignment to be a DatasetTable"); + } + } +} From d01879ea611baf97b169f0e4bc0da28ac1260887 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 Jun 2025 10:53:43 -0700 Subject: [PATCH 17/52] Add query for genomic data --- SivStudies/resources/etls/siv-studies.xml | 26 +++++++++++++++++-- .../queries/study/genomicsDataSource.sql | 15 +++++++++++ .../scRNAseqDataSource.sql} | 0 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 SivStudies/resources/queries/study/genomicsDataSource.sql rename SivStudies/resources/queries/{singlecell/sivDataSource.sql => study/scRNAseqDataSource.sql} (100%) diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml index 6bb6fc2d..0de3dd60 100644 --- a/SivStudies/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -145,8 +145,8 @@ - - + + @@ -158,6 +158,28 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/SivStudies/resources/queries/study/genomicsDataSource.sql b/SivStudies/resources/queries/study/genomicsDataSource.sql new file mode 100644 index 00000000..bb332a14 --- /dev/null +++ b/SivStudies/resources/queries/study/genomicsDataSource.sql @@ -0,0 +1,15 @@ +SELECT + +s1.subjectId as Id, +s1.date, + +('Assay(s): ' || librarytype) as description + +FROM (SELECT + s.subjectId, + coalesce(s.sampleDate, now()) as date, + GROUP_CONCAT(DISTINCT s.librarytype, ', ') as librarytype + + FROM "/Internal/ColonyData".sequenceanalysis.sequence_readsets s + GROUP BY s.subjectId, s.sampleDate +) s1 \ No newline at end of file diff --git a/SivStudies/resources/queries/singlecell/sivDataSource.sql b/SivStudies/resources/queries/study/scRNAseqDataSource.sql similarity index 100% rename from SivStudies/resources/queries/singlecell/sivDataSource.sql rename to SivStudies/resources/queries/study/scRNAseqDataSource.sql From 5eb96613eafda5fec898c54ba13ad9a85ac754fa Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 Jun 2025 13:03:52 -0700 Subject: [PATCH 18/52] Add flow dataset --- .../resources/queries/study/flow.query.xml | 34 ++++++++++++++++++ .../study/datasets/datasets_manifest.xml | 3 ++ .../study/datasets/datasets_metadata.xml | 35 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 SivStudies/resources/queries/study/flow.query.xml diff --git a/SivStudies/resources/queries/study/flow.query.xml b/SivStudies/resources/queries/study/flow.query.xml new file mode 100644 index 00000000..5f79cd06 --- /dev/null +++ b/SivStudies/resources/queries/study/flow.query.xml @@ -0,0 +1,34 @@ + + + + + Flow Cytometry + + + + Start Date + Date + + + Sample Type + + + Assay Type + + + Population + + + Result + + + Units + + + Comments + + +
+ + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml index 5361e959..e1816813 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml @@ -40,5 +40,8 @@ + + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index 0f0e3fe5..7b841d62 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -414,4 +414,39 @@ Additional Datatypes + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + varchar + + + varchar + + + varchar + + + double + + + varchar + + + varchar + + + Flow Cytometry +
From 1ca30732c45f14b5dd08199f3b71e6da570f305b Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 Jun 2025 16:39:28 -0700 Subject: [PATCH 19/52] Make IDs unique in ETL --- SivStudies/resources/etls/siv-studies.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml index 0de3dd60..aed74306 100644 --- a/SivStudies/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -159,7 +159,7 @@ - + From c999935d2a63323eb6210e8adc01385f3bcee6e3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 Jun 2025 16:44:01 -0700 Subject: [PATCH 20/52] Set conceptURIs --- SivStudies/resources/queries/study/chemistryPivot.query.xml | 1 + SivStudies/resources/queries/study/hematologyPivot.query.xml | 1 + SivStudies/resources/queries/study/studyData.query.xml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/SivStudies/resources/queries/study/chemistryPivot.query.xml b/SivStudies/resources/queries/study/chemistryPivot.query.xml index 4ff5a950..4146eac0 100644 --- a/SivStudies/resources/queries/study/chemistryPivot.query.xml +++ b/SivStudies/resources/queries/study/chemistryPivot.query.xml @@ -6,6 +6,7 @@ + http://cpas.labkey.com/Study#ParticipantId study animal diff --git a/SivStudies/resources/queries/study/hematologyPivot.query.xml b/SivStudies/resources/queries/study/hematologyPivot.query.xml index 6ae5b45c..7daa8ac3 100644 --- a/SivStudies/resources/queries/study/hematologyPivot.query.xml +++ b/SivStudies/resources/queries/study/hematologyPivot.query.xml @@ -6,6 +6,7 @@ + http://cpas.labkey.com/Study#ParticipantId study animal diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml index bf1d995a..f6a3f85e 100644 --- a/SivStudies/resources/queries/study/studyData.query.xml +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -5,7 +5,7 @@ - + http://cpas.labkey.com/Study#ParticipantId Date From 39a66c2e173ab0bd7f705e57e0b1ec48c5e1ea22 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 Jun 2025 16:57:05 -0700 Subject: [PATCH 21/52] Update view --- tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml b/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml index 332f09b6..d07972bd 100644 --- a/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml +++ b/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml @@ -6,6 +6,7 @@ + From 96e433d1d238392f5406ea3ea7abef4471565876 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 21 Jun 2025 10:33:31 -0700 Subject: [PATCH 22/52] Add stubs for studies module customizer --- .../resources/queries/study/studyData.query.xml | 14 ++++++++++++++ .../sivstudies/query/SivStudiesCustomizer.java | 3 +++ 2 files changed, 17 insertions(+) diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml index f6a3f85e..03f6dc22 100644 --- a/SivStudies/resources/queries/study/studyData.query.xml +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -19,6 +19,20 @@ false + + + + + + diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index e9133949..29755e23 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -9,6 +9,7 @@ import org.labkey.api.data.SQLFragment; import org.labkey.api.data.TableInfo; import org.labkey.api.data.WrappedColumn; +import org.labkey.api.ldk.LDKService; import org.labkey.api.ldk.table.AbstractTableCustomizer; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.LookupForeignKey; @@ -18,6 +19,7 @@ import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; import org.labkey.api.security.User; +import org.labkey.api.studies.StudiesService; import org.labkey.api.study.Dataset; import org.labkey.api.study.DatasetTable; import org.labkey.api.util.logging.LogHelper; @@ -35,6 +37,7 @@ public class SivStudiesCustomizer extends AbstractTableCustomizer @Override public void customize(TableInfo tableInfo) { + StudiesService.get().getStudiesTableCustomizer().customize(tableInfo); if (tableInfo instanceof DatasetTable ds) { performDatasetCustomization(ds); From f702fe9bc90f60660eb68cf3e8a85594d818303b Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 22 Jun 2025 16:06:24 -0700 Subject: [PATCH 23/52] Add stubs for studies webpart --- .../folderTypes/SIV Studies.folderType.xml | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml index 0f25bbe0..98951bf2 100644 --- a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml +++ b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml @@ -2,33 +2,17 @@ SIV Studies Overview The default folder layout for Studies - - - - - - - - - - - - - - - datasets - Datasets + overview + Overview - - datasets - + - Datasets + Studies Overview body From 8eadc22ce1f354422c9460c74a73ce9620917a17 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 22 Jun 2025 20:58:09 -0700 Subject: [PATCH 24/52] Update key fields --- SivStudies/resources/queries/study/demographics.query.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/SivStudies/resources/queries/study/demographics.query.xml b/SivStudies/resources/queries/study/demographics.query.xml index 5c994581..3125d9f0 100644 --- a/SivStudies/resources/queries/study/demographics.query.xml +++ b/SivStudies/resources/queries/study/demographics.query.xml @@ -6,7 +6,6 @@ - true true @@ -33,7 +32,6 @@ true false - true Sex From 6950426b8cd809b6aaf6ca31ad1b060ad0bd5b0f Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 23 Jun 2025 07:55:16 -0700 Subject: [PATCH 25/52] Add new dataset and perform code cleanup --- SivStudies/resources/data/reports.tsv | 35 ++++++++++--------- .../queries/study/outcomes.query.xml | 28 +++++++++++++++ .../queries/study/studyData.query.xml | 3 -- .../study/datasets/datasets_manifest.xml | 3 ++ .../study/datasets/datasets_metadata.xml | 26 ++++++++++++++ .../BlastPipelineJobResourceAllocator.java | 2 +- .../SequenceJobResourceAllocator.java | 22 ++++++------ 7 files changed, 88 insertions(+), 31 deletions(-) create mode 100644 SivStudies/resources/queries/study/outcomes.query.xml diff --git a/SivStudies/resources/data/reports.tsv b/SivStudies/resources/data/reports.tsv index bae6925e..207af3a4 100644 --- a/SivStudies/resources/data/reports.tsv +++ b/SivStudies/resources/data/reports.tsv @@ -1,16 +1,19 @@ -reporttype category schemaname queryname viewname reporttitle containerpath -query General study additionalDatatypes Additional Data -query Clinical study weight Weights -query Clinical study treatments Treatments/Medications -query Clinical study chemistryPivot Blood Chemistry -query Clinical study hematologyPivot CBC/Hematology -query Research study assignment Project/Study Assignment -query Research study genetics Genetic Data / MHC -query Clinical study procedures Procedures -query Research study viralLoads Viral Loads -query General study samples Samples -query Research study immunizations Immunizations -query General study flags Flags/Misc Information -query General study demographics Demographics -query General study demographics Project Summary Project Summary -query General study demographics MHC Type MHC Typing +reporttype category schemaname queryname viewname reporttitle subjectfieldname +query General study additionalDatatypes Additional Data +query Clinical study weight Weights +query Clinical study treatments Treatments/Medications +query Clinical study chemistryPivot Blood Chemistry +query Clinical study hematologyPivot CBC/Hematology +query Study Design study assignment Project/Study Assignment +query Research study flow Flow Cytometry +query Research study outcomes Study Outcomes/Phenotypes +query Research study genetics Genetic Data / MHC +query Clinical study procedures Procedures +query Research study viralLoads Viral Loads +query General study samples Samples +query Research study immunizations Immunizations +query General study flags Flags/Misc Information +query General study demographics Demographics +query General study demographics Project Summary Project Summary +query General study demographics MHC Type MHC Typing +query Study Design studies timepointToDate Timepoints subjectId diff --git a/SivStudies/resources/queries/study/outcomes.query.xml b/SivStudies/resources/queries/study/outcomes.query.xml new file mode 100644 index 00000000..038f6284 --- /dev/null +++ b/SivStudies/resources/queries/study/outcomes.query.xml @@ -0,0 +1,28 @@ + + + + + + + + Date + Date + + + Outcome/Phenotype + + + Sub-Outcome + + + Comments + + + Key + true + + +
+ + + diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml index 03f6dc22..18418028 100644 --- a/SivStudies/resources/queries/study/studyData.query.xml +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -26,9 +26,6 @@ - diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml index e1816813..715134ec 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml @@ -43,5 +43,8 @@ + + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index 7b841d62..29fa3a75 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -449,4 +449,30 @@ Flow Cytometry + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + varchar + + + varchar + + + varchar + + + Outcomes/Phenotypes +
diff --git a/primeseq/src/org/labkey/primeseq/pipeline/BlastPipelineJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/BlastPipelineJobResourceAllocator.java index 73e1ab10..7f2a687e 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/BlastPipelineJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/BlastPipelineJobResourceAllocator.java @@ -40,7 +40,7 @@ public Integer getMaxRequestMemory(PipelineJob job) } @Override - public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine engine, List lines) + public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine engine, List lines) { //force BLAST jobs to top of queue, since we assume these run quickly if ("HTCondorEngine".equals(engine.getType())) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index 1151b766..c213266f 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -116,8 +116,8 @@ public Integer getMaxRequestCpus(PipelineJob job) if (isSequenceNormalizationTask(job)) { - job.getLogger().debug("setting max CPUs to 4"); - return 4; + job.getLogger().debug("setting max CPUs to 2"); + return 2; } if (isLuceneIndexJob(job)) @@ -137,15 +137,15 @@ public Integer getMaxRequestCpus(PipelineJob job) //10gb if (totalFileSize < 10e9) { - job.getLogger().debug("file size less than 10gb, lowering CPUs to 8"); + job.getLogger().debug("file size less than 10gb, lowering CPUs to 4"); - return 8; + return 4; } else if (totalFileSize < 20e9) { - job.getLogger().debug("file size less than 20gb, lowering CPUs to 12"); + job.getLogger().debug("file size less than 20gb, lowering CPUs to 8"); - return 12; + return 8; } job.getLogger().debug("file size greater than 20gb, using 12 CPUs"); @@ -318,7 +318,7 @@ public Integer getMaxRequestMemory(PipelineJob job) } @Override - public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine engine, List lines) + public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine engine, List lines) { if (job instanceof HasJobParams) { @@ -332,7 +332,7 @@ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine eng } @Override - public @NotNull Map getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine) + public @NotNull Map getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine) { Map ret = new HashMap<>(); @@ -358,7 +358,7 @@ private String getTime(PipelineJob job) return null; } - private void possiblyAddHighIO(PipelineJob job, RemoteExecutionEngine engine, List lines) + private void possiblyAddHighIO(PipelineJob job, RemoteExecutionEngine engine, List lines) { Map params = ((HasJobParams)job).getJobParams(); String val = StringUtils.trimToNull(params.get("resourceSettings.resourceSettings.highIO")); @@ -522,7 +522,7 @@ private void possiblyAddQOS(PipelineJob job, RemoteExecutionEngine engine, List< job.getLogger().debug("qos as supplied by job: " + qos); //Exempt specific task types: - TaskFactory factory = job.getActiveTaskFactory(); + TaskFactory factory = job.getActiveTaskFactory(); String activeTask = factory == null ? null : factory.getId().getNamespaceClass().getSimpleName(); job.getLogger().debug("Active task simplename: " + activeTask); if (Arrays.asList("PrepareAlignerIndexesTask", "CacheAlignerIndexesTask", "AlignmentInitTask", "VariantProcessingScatterRemotePrepareTask").contains(activeTask)) @@ -610,7 +610,7 @@ private Long getFileSize(PipelineJob job) } @Override - public void processJavaOpts(PipelineJob job, RemoteExecutionEngine engine, @NotNull List existingJavaOpts) + public void processJavaOpts(PipelineJob job, RemoteExecutionEngine engine, @NotNull List existingJavaOpts) { if (job instanceof HasJobParams) { From b1fb0d40829b0b33c217df8765b1955d281a2e68 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 23 Jun 2025 10:15:04 -0700 Subject: [PATCH 26/52] Switch from librarytype to application in genomics query --- SivStudies/resources/queries/study/genomicsDataSource.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SivStudies/resources/queries/study/genomicsDataSource.sql b/SivStudies/resources/queries/study/genomicsDataSource.sql index bb332a14..cb876789 100644 --- a/SivStudies/resources/queries/study/genomicsDataSource.sql +++ b/SivStudies/resources/queries/study/genomicsDataSource.sql @@ -3,12 +3,12 @@ SELECT s1.subjectId as Id, s1.date, -('Assay(s): ' || librarytype) as description +('Assay(s): ' || application) as description FROM (SELECT s.subjectId, coalesce(s.sampleDate, now()) as date, - GROUP_CONCAT(DISTINCT s.librarytype, ', ') as librarytype + GROUP_CONCAT(DISTINCT s.application, ', ') as application FROM "/Internal/ColonyData".sequenceanalysis.sequence_readsets s GROUP BY s.subjectId, s.sampleDate From c0b7da17db24ce3d634ad99afb40a5f4f37c54c4 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 23 Jun 2025 11:44:53 -0700 Subject: [PATCH 27/52] Correct label --- .../resources/queries/study/additionalDatatypes.query.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SivStudies/resources/queries/study/additionalDatatypes.query.xml b/SivStudies/resources/queries/study/additionalDatatypes.query.xml index 1fdf4468..953d23c6 100644 --- a/SivStudies/resources/queries/study/additionalDatatypes.query.xml +++ b/SivStudies/resources/queries/study/additionalDatatypes.query.xml @@ -13,7 +13,7 @@ Category - Procedure + Description From 482303fba18bc3b3f41983fe817abcdd879e490b Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 23 Jun 2025 12:46:25 -0700 Subject: [PATCH 28/52] Add custom views --- .../clone_responses/Cohort Info.qview.xml | 24 +++++++++++++++++++ .../queries/tcrdb/stims/Cohort Info.qview.xml | 21 ++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tcrdb/resources/queries/tcrdb/clone_responses/Cohort Info.qview.xml create mode 100644 tcrdb/resources/queries/tcrdb/stims/Cohort Info.qview.xml diff --git a/tcrdb/resources/queries/tcrdb/clone_responses/Cohort Info.qview.xml b/tcrdb/resources/queries/tcrdb/clone_responses/Cohort Info.qview.xml new file mode 100644 index 00000000..4e7d72c8 --- /dev/null +++ b/tcrdb/resources/queries/tcrdb/clone_responses/Cohort Info.qview.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/stims/Cohort Info.qview.xml b/tcrdb/resources/queries/tcrdb/stims/Cohort Info.qview.xml new file mode 100644 index 00000000..2749f5cd --- /dev/null +++ b/tcrdb/resources/queries/tcrdb/stims/Cohort Info.qview.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 1be95d20d11ff592df2661ab15575bf95c5c7bd7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 12:13:32 -0700 Subject: [PATCH 29/52] Refinements to customizer code, and many new SIV-related ETLs --- .../queries/bimber_data/idrOutcomeSource.sql | 14 +++ .../queries/bimber_data/idrPvlSource.sql | 11 ++ .../queries/bimber_data/idrSampleSource.sql | 27 ++++ SivStudies/resources/etls/idr-data.xml | 117 ++++++++++++++++++ SivStudies/resources/etls/siv-studies.xml | 13 +- .../study/additionalDatatypes.query.xml | 3 + .../queries/study/assignment.query.xml | 3 + .../queries/study/demographics.query.xml | 5 + .../resources/queries/study/flags.query.xml | 3 + .../queries/study/genetics.query.xml | 3 + .../queries/study/genomicsDataSource.sql | 3 +- .../queries/study/immunizations.query.xml | 3 + .../resources/queries/study/labwork.query.xml | 3 + .../queries/study/outcomes.query.xml | 3 + .../queries/study/procedures.query.xml | 3 + .../resources/queries/study/samples.query.xml | 4 +- .../queries/study/scRNAseqDataSource.sql | 3 +- .../queries/study/studyData.query.xml | 1 + .../queries/study/treatments.query.xml | 3 + .../queries/study/viralloads.query.xml | 3 + .../resources/queries/study/weight.query.xml | 3 + .../study/datasets/datasets_metadata.xml | 42 +++++++ .../sivstudies/etl/SubjectScopedSelect.java | 45 ++++++- .../query/SivStudiesCustomizer.java | 9 ++ .../query/ViralLoadsTriggerFactory.java | 89 +++++++++++++ 25 files changed, 402 insertions(+), 14 deletions(-) create mode 100644 IDR/resources/queries/bimber_data/idrOutcomeSource.sql create mode 100644 IDR/resources/queries/bimber_data/idrPvlSource.sql create mode 100644 IDR/resources/queries/bimber_data/idrSampleSource.sql create mode 100644 SivStudies/resources/etls/idr-data.xml create mode 100644 SivStudies/src/org/labkey/sivstudies/query/ViralLoadsTriggerFactory.java diff --git a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql new file mode 100644 index 00000000..7621857c --- /dev/null +++ b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql @@ -0,0 +1,14 @@ +SELECT + +Rh as Id, +cohortStart as date, + +CASE + WHEN contprog = 'C' THEN 'Controller' + WHEN contprog = 'P' THEN 'Progressor' +END as outcome, + +'Hansen/IDR' as dataSource + +FROM bimber_data.subjects +WHERE contprog IS NOT NULL AND contprog != '' \ No newline at end of file diff --git a/IDR/resources/queries/bimber_data/idrPvlSource.sql b/IDR/resources/queries/bimber_data/idrPvlSource.sql new file mode 100644 index 00000000..5ff9a748 --- /dev/null +++ b/IDR/resources/queries/bimber_data/idrPvlSource.sql @@ -0,0 +1,11 @@ +SELECT +i_rh as Id, +d_PVLDate as date, +c_PVL as result, + +'Copies/mL' as units, +'Plasma' as sampleType, +'SIVmac239' as target, +'Hansen/IDR' as dataSource + +FROM bimber_data.pvl \ No newline at end of file diff --git a/IDR/resources/queries/bimber_data/idrSampleSource.sql b/IDR/resources/queries/bimber_data/idrSampleSource.sql new file mode 100644 index 00000000..0a38d53a --- /dev/null +++ b/IDR/resources/queries/bimber_data/idrSampleSource.sql @@ -0,0 +1,27 @@ +SELECT + +MonkeyId as Id, +ID as sampleid, +SampleDate as date, +Tissue as sampleType, +CellCnt as quantity, + +'Hansen/IDR' as dataSource + +FROM bimber_data.ln_loc +WHERE SampleDate IS NOT NULL + +UNION ALL + +SELECT + +Rh as Id, +ID as sampleid, +SampleDate as date, +Tissue as sampleType, +null as quantity, + +'Hansen/IDR' as dataSource + +FROM bimber_data.ult_loc +WHERE SampleDate IS NOT NULL \ No newline at end of file diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml new file mode 100644 index 00000000..dbd35bb7 --- /dev/null +++ b/SivStudies/resources/etls/idr-data.xml @@ -0,0 +1,117 @@ + + + Hansen/IDR + SIV Studies / Hansen/IDR Data + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml index aed74306..c99ffd98 100644 --- a/SivStudies/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -37,6 +37,7 @@ + @@ -58,7 +59,7 @@ - + @@ -81,7 +82,7 @@ - + @@ -105,7 +106,7 @@ - + @@ -129,10 +130,12 @@ + + @@ -148,7 +151,7 @@ - + @@ -170,7 +173,7 @@ - + diff --git a/SivStudies/resources/queries/study/additionalDatatypes.query.xml b/SivStudies/resources/queries/study/additionalDatatypes.query.xml index 953d23c6..d14427e2 100644 --- a/SivStudies/resources/queries/study/additionalDatatypes.query.xml +++ b/SivStudies/resources/queries/study/additionalDatatypes.query.xml @@ -12,6 +12,9 @@ Category + + Data Source + Description diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml index 8ce5db9f..66e945ed 100644 --- a/SivStudies/resources/queries/study/assignment.query.xml +++ b/SivStudies/resources/queries/study/assignment.query.xml @@ -21,6 +21,9 @@ Cohort ID + + Data Source + Category diff --git a/SivStudies/resources/queries/study/demographics.query.xml b/SivStudies/resources/queries/study/demographics.query.xml index 3125d9f0..d9d6914f 100644 --- a/SivStudies/resources/queries/study/demographics.query.xml +++ b/SivStudies/resources/queries/study/demographics.query.xml @@ -6,6 +6,7 @@ + true true @@ -31,6 +32,7 @@ true + true false @@ -57,6 +59,9 @@ Status + + Data Source + diff --git a/SivStudies/resources/queries/study/flags.query.xml b/SivStudies/resources/queries/study/flags.query.xml index d8795702..91419bce 100644 --- a/SivStudies/resources/queries/study/flags.query.xml +++ b/SivStudies/resources/queries/study/flags.query.xml @@ -15,6 +15,9 @@ Flag + + Data Source + diff --git a/SivStudies/resources/queries/study/genetics.query.xml b/SivStudies/resources/queries/study/genetics.query.xml index d29cc2b1..e0aac8e5 100644 --- a/SivStudies/resources/queries/study/genetics.query.xml +++ b/SivStudies/resources/queries/study/genetics.query.xml @@ -23,6 +23,9 @@ Score + + Data Source + diff --git a/SivStudies/resources/queries/study/genomicsDataSource.sql b/SivStudies/resources/queries/study/genomicsDataSource.sql index cb876789..38d6b445 100644 --- a/SivStudies/resources/queries/study/genomicsDataSource.sql +++ b/SivStudies/resources/queries/study/genomicsDataSource.sql @@ -3,7 +3,8 @@ SELECT s1.subjectId as Id, s1.date, -('Assay(s): ' || application) as description +('Assay(s): ' || application) as description, +'ONPRC Genetics Core' as dataSource FROM (SELECT s.subjectId, diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml index 06a2c2f7..688745a5 100644 --- a/SivStudies/resources/queries/study/immunizations.query.xml +++ b/SivStudies/resources/queries/study/immunizations.query.xml @@ -26,6 +26,9 @@ Reason + + Data Source + diff --git a/SivStudies/resources/queries/study/labwork.query.xml b/SivStudies/resources/queries/study/labwork.query.xml index 7f411731..4bbfcf11 100644 --- a/SivStudies/resources/queries/study/labwork.query.xml +++ b/SivStudies/resources/queries/study/labwork.query.xml @@ -34,6 +34,9 @@ Method + + Data Source + diff --git a/SivStudies/resources/queries/study/outcomes.query.xml b/SivStudies/resources/queries/study/outcomes.query.xml index 038f6284..e655eb67 100644 --- a/SivStudies/resources/queries/study/outcomes.query.xml +++ b/SivStudies/resources/queries/study/outcomes.query.xml @@ -17,6 +17,9 @@ Comments + + Data Source + Key true diff --git a/SivStudies/resources/queries/study/procedures.query.xml b/SivStudies/resources/queries/study/procedures.query.xml index b866f8b5..6624d04d 100644 --- a/SivStudies/resources/queries/study/procedures.query.xml +++ b/SivStudies/resources/queries/study/procedures.query.xml @@ -15,6 +15,9 @@ Procedure + + Data Source + diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml index a0fef565..922600f5 100644 --- a/SivStudies/resources/queries/study/samples.query.xml +++ b/SivStudies/resources/queries/study/samples.query.xml @@ -23,7 +23,9 @@ Quantity Units - + + Data Source + diff --git a/SivStudies/resources/queries/study/scRNAseqDataSource.sql b/SivStudies/resources/queries/study/scRNAseqDataSource.sql index 1cf6eab1..2517b709 100644 --- a/SivStudies/resources/queries/study/scRNAseqDataSource.sql +++ b/SivStudies/resources/queries/study/scRNAseqDataSource.sql @@ -5,7 +5,8 @@ s1.date, ('Assay(s): ' || assayType || char(10) || 'Tissue(s): ' || tissue || char(10) || -'Stims(s): ' || stims || char(10)) as description +'Stims(s): ' || stims || char(10)) as description, +'Bimber Lab' as dataSource FROM (SELECT s.subjectId, diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml index 18418028..8522b6c9 100644 --- a/SivStudies/resources/queries/study/studyData.query.xml +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -16,6 +16,7 @@ Key true + true false diff --git a/SivStudies/resources/queries/study/treatments.query.xml b/SivStudies/resources/queries/study/treatments.query.xml index 1a5e82c9..cc710853 100644 --- a/SivStudies/resources/queries/study/treatments.query.xml +++ b/SivStudies/resources/queries/study/treatments.query.xml @@ -65,6 +65,9 @@ true false + + Data Source + diff --git a/SivStudies/resources/queries/study/viralloads.query.xml b/SivStudies/resources/queries/study/viralloads.query.xml index dcef9ed6..df20996f 100644 --- a/SivStudies/resources/queries/study/viralloads.query.xml +++ b/SivStudies/resources/queries/study/viralloads.query.xml @@ -41,6 +41,9 @@ true false + + Data Source + diff --git a/SivStudies/resources/queries/study/weight.query.xml b/SivStudies/resources/queries/study/weight.query.xml index 24e484dd..97a6f737 100644 --- a/SivStudies/resources/queries/study/weight.query.xml +++ b/SivStudies/resources/queries/study/weight.query.xml @@ -13,6 +13,9 @@ Weight (kg) 0.#### + + Data Source + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index 29fa3a75..c45b36e2 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -10,6 +10,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -34,6 +37,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -94,6 +100,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -130,6 +139,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -151,6 +163,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -197,6 +212,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -235,6 +253,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid false @@ -277,6 +298,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -313,6 +337,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -346,6 +373,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -378,6 +408,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -401,6 +434,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -424,6 +460,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true @@ -459,6 +498,9 @@ timestamp http://cpas.labkey.com/laboratory#sampleDate + + varchar + entityid true diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index c7f1c754..c3db2a86 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -354,7 +354,21 @@ private List> getRowsToImport(List subjects, Logger if (_settings.get(Settings.dataRemoteSource.name()) != null) { - DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.dataRemoteSource.name()), log); + Container target; + if (_settings.get(Settings.dataSourceContainerPath.name()) != null) + { + target = ContainerManager.getForPath(_settings.get(Settings.dataSourceContainerPath.name())); + if (target == null) + { + throw new IllegalStateException("Unknown container: " + _settings.get(Settings.dataSourceContainerPath.name())); + } + } + else + { + target =_containerUser.getContainer(); + } + + DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.dataRemoteSource.name()), target, log); SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.dataSourceSchema.name()), _settings.get(Settings.dataSourceQuery.name())); sr.setColumns(sourceColumns); sr.addFilter(_settings.get(Settings.dataSourceSubjectColumn.name()), StringUtils.join(subjects, ";"), Filter.Operator.IN); @@ -400,11 +414,16 @@ else if (f.getParamVals().length == 1) } else { + Container source; if (_settings.get(Settings.dataSourceContainerPath.name()) == null) { - throw new IllegalStateException("Must provide dataSourceContainerPath for local sources"); + source = _containerUser.getContainer(); + } + else + { + source = ContainerManager.getForPath(_settings.get(Settings.dataSourceContainerPath.name())); } - Container source = ContainerManager.getForPath(_settings.get(Settings.dataSourceContainerPath.name())); + if (source == null) { throw new IllegalStateException("Unknown container: " + _settings.get(Settings.dataSourceContainerPath.name())); @@ -543,9 +562,9 @@ public void setSettings(Map settings) _settings.putAll(settings); } - private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Logger log) throws IllegalStateException + private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Container c, Logger log) throws IllegalStateException { - DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, _containerUser.getContainer(), log); + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, c, log); if (rc == null) { throw new IllegalStateException("Unable to find remote connection: " + name); @@ -558,7 +577,21 @@ private List getSubjects(Logger log) { if (_settings.get(Settings.subjectRemoteSource.name()) != null) { - DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.subjectRemoteSource.name()), log); + Container target; + if (_settings.get(Settings.subjectSourceContainerPath.name()) != null) + { + target = ContainerManager.getForPath(_settings.get(Settings.subjectSourceContainerPath.name())); + if (target == null) + { + throw new IllegalStateException("Unknown container: " + _settings.get(Settings.subjectSourceContainerPath.name())); + } + } + else + { + target =_containerUser.getContainer(); + } + + DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.subjectRemoteSource.name()), target, log); SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.subjectSourceSchema.name()), _settings.get(Settings.subjectSourceQuery.name())); sr.setColumns(Arrays.asList(_settings.get(Settings.subjectSourceColumn.name()))); diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 29755e23..02e4f31d 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -60,6 +60,10 @@ public void performDatasetCustomization(DatasetTable ds) { appendDemographicsColumns(ati); } + else if ("viralLoads".equalsIgnoreCase(ds.getName())) + { + customizeViralLoads(ati); + } } else { @@ -260,4 +264,9 @@ private BaseColumnInfo getWrappedIdCol(UserSchema targetQueryUserSchema, String return col; } + + private void customizeViralLoads(AbstractTableInfo ati) + { + ati.addTriggerFactory(new ViralLoadsTriggerFactory()); + } } diff --git a/SivStudies/src/org/labkey/sivstudies/query/ViralLoadsTriggerFactory.java b/SivStudies/src/org/labkey/sivstudies/query/ViralLoadsTriggerFactory.java new file mode 100644 index 00000000..7f9bb267 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/query/ViralLoadsTriggerFactory.java @@ -0,0 +1,89 @@ +package org.labkey.sivstudies.query; + +import org.apache.commons.lang3.math.NumberUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.triggers.Trigger; +import org.labkey.api.data.triggers.TriggerFactory; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class ViralLoadsTriggerFactory implements TriggerFactory +{ + public ViralLoadsTriggerFactory() + { + + } + + @Override + public @NotNull Collection createTrigger(@Nullable Container c, TableInfo table, Map extraContext) + { + return List.of(new ViralLoadTrigger()); + } + + public static class ViralLoadTrigger implements Trigger + { + @Override + public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map newRow, ValidationException errors, Map extraContext) throws ValidationException + { + beforeInsert(table, c, user, newRow, errors, extraContext, null); + } + + @Override + public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map newRow, ValidationException errors, Map extraContext, @Nullable Map existingRecord) throws ValidationException + { + handlePVL(newRow, c, errors); + } + + @Override + public void beforeUpdate(TableInfo table, Container c, User user, @Nullable Map newRow, @Nullable Map oldRow, ValidationException errors, Map extraContext) throws ValidationException + { + handlePVL(newRow, c, errors); + } + + /** + * This allows incoming data to specify the study using the string name, which is resolved into the rowId + */ + private void handlePVL(@Nullable Map row, Container c, ValidationException errors) + { + if (row == null) + { + return; + } + + if (row.get("result") == null || NumberUtils.isCreatable(row.get("result").toString())) + { + return; + } + + String val = row.get("result").toString(); + val = val.replaceAll(",", ""); + if (NumberUtils.isCreatable(val)) + { + row.put("result", val); + } + else if (val.startsWith("Below")) + { + val = val.replace("Below ", ""); + if (NumberUtils.isCreatable(val)) + { + row.put("result", val); + row.put("resultOORIndicator", "<"); + row.put("lod", val); + } + } + + if (!NumberUtils.isCreatable(val)) + { + errors.addError(new SimpleValidationError("Non-numeric VL: "+ val, "result", ValidationException.SEVERITY.ERROR)); + } + } + } +} From d54b495f85cc031e473e49b62373c8ed062151d7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 12:20:44 -0700 Subject: [PATCH 30/52] Switch field value --- IDR/resources/queries/bimber_data/idrPvlSource.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IDR/resources/queries/bimber_data/idrPvlSource.sql b/IDR/resources/queries/bimber_data/idrPvlSource.sql index 5ff9a748..cd326b9f 100644 --- a/IDR/resources/queries/bimber_data/idrPvlSource.sql +++ b/IDR/resources/queries/bimber_data/idrPvlSource.sql @@ -5,7 +5,8 @@ c_PVL as result, 'Copies/mL' as units, 'Plasma' as sampleType, -'SIVmac239' as target, +'SIVmac239' as assayType, +'SIV' as target, 'Hansen/IDR' as dataSource FROM bimber_data.pvl \ No newline at end of file From a41a968b7f15fbd55eed429787053b0b82f507cc Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 12:32:48 -0700 Subject: [PATCH 31/52] Expand ETL --- SivStudies/resources/etls/idr-data.xml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml index dbd35bb7..dc9d527c 100644 --- a/SivStudies/resources/etls/idr-data.xml +++ b/SivStudies/resources/etls/idr-data.xml @@ -110,6 +110,29 @@ + + + + + + + + + + + + + + + + + + + + + + + From 1c32d8fbf9d2bc2d02d3a3852aa94852631f1a73 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 12:58:00 -0700 Subject: [PATCH 32/52] Expand SIV Queries --- SivStudies/resources/data/reports.tsv | 2 ++ .../study/demographics/Expanded.qview.xml | 17 ++++++++++++++ .../study/demographicsImmunizations.query.xml | 22 +++++++++++++++++++ .../study/demographicsImmunizations.sql | 7 ++++++ .../study/demographicsOutcomes.query.xml | 19 ++++++++++++++++ .../queries/study/demographicsOutcomes.sql | 6 +++++ .../queries/study/demographicsProjects.sql | 2 +- .../query/SivStudiesCustomizer.java | 6 ++--- 8 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 SivStudies/resources/queries/study/demographics/Expanded.qview.xml create mode 100644 SivStudies/resources/queries/study/demographicsImmunizations.query.xml create mode 100644 SivStudies/resources/queries/study/demographicsImmunizations.sql create mode 100644 SivStudies/resources/queries/study/demographicsOutcomes.query.xml create mode 100644 SivStudies/resources/queries/study/demographicsOutcomes.sql diff --git a/SivStudies/resources/data/reports.tsv b/SivStudies/resources/data/reports.tsv index 207af3a4..65f6076a 100644 --- a/SivStudies/resources/data/reports.tsv +++ b/SivStudies/resources/data/reports.tsv @@ -16,4 +16,6 @@ query General study flags Flags/Misc Information query General study demographics Demographics query General study demographics Project Summary Project Summary query General study demographics MHC Type MHC Typing +query General study demographics Expanded Expanded Information query Study Design studies timepointToDate Timepoints subjectId +query Study Design studies subjectAnchorDates Anchor Dates subjectId diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml new file mode 100644 index 00000000..6cfb780d --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml @@ -0,0 +1,17 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsImmunizations.query.xml b/SivStudies/resources/queries/study/demographicsImmunizations.query.xml new file mode 100644 index 00000000..20e1fe98 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsImmunizations.query.xml @@ -0,0 +1,22 @@ + + + + + Immunization Summary + + + true + true + + + Immunizations + + + Immunization Types + + + immunizations +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsImmunizations.sql b/SivStudies/resources/queries/study/demographicsImmunizations.sql new file mode 100644 index 00000000..a31a58ce --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsImmunizations.sql @@ -0,0 +1,7 @@ +SELECT + s.Id, + group_concat(DISTINCT s.treatment, char(10)) as immunizations, + group_concat(DISTINCT s.category, char(10)) as immunizationTypes, + +FROM study.immunizations s +GROUP BY s.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsOutcomes.query.xml b/SivStudies/resources/queries/study/demographicsOutcomes.query.xml new file mode 100644 index 00000000..22acc4fa --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsOutcomes.query.xml @@ -0,0 +1,19 @@ + + + + + Outcome Summary + + + true + true + + + Outcomes + + + outcomes +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsOutcomes.sql b/SivStudies/resources/queries/study/demographicsOutcomes.sql new file mode 100644 index 00000000..22ffd948 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsOutcomes.sql @@ -0,0 +1,6 @@ +SELECT + s.Id, + group_concat(DISTINCT s.outcome, char(10)) as outcomes + +FROM study.outcomes s +GROUP BY s.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsProjects.sql b/SivStudies/resources/queries/study/demographicsProjects.sql index 0a6666c7..e01f2a07 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.sql +++ b/SivStudies/resources/queries/study/demographicsProjects.sql @@ -1,6 +1,6 @@ SELECT s.Id, - count(s.Id) as totalTests, + count(s.Id) as totalProjects, group_concat(DISTINCT s.study, char(10)) as allStudies, group_concat(DISTINCT s.category, char(10)) as categories, diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 02e4f31d..cb8686a3 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -218,10 +218,10 @@ private void appendDemographicsColumns(AbstractTableInfo demographicsTable) demographicsTable.addColumn(colInfo); } - if (demographicsTable.getColumn("challenges") == null) + if (demographicsTable.getColumn("outcomes") == null) { - BaseColumnInfo colInfo = getWrappedIdCol(demographicsTable.getUserSchema(), "demographicsChallenges", demographicsTable, "challenges"); - colInfo.setLabel("Challenge Summary"); + BaseColumnInfo colInfo = getWrappedIdCol(demographicsTable.getUserSchema(), "demographicsOutcomes", demographicsTable, "outcomes"); + colInfo.setLabel("Outcomes"); demographicsTable.addColumn(colInfo); } } From f6348db51ca6648e16cf4b379cb8e2526d193eb5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 13:05:11 -0700 Subject: [PATCH 33/52] Expand ETL --- .../bimber_data/idrAnchorDateSource.sql | 21 +++++++++++++++++++ SivStudies/resources/etls/idr-data.xml | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 IDR/resources/queries/bimber_data/idrAnchorDateSource.sql diff --git a/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql b/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql new file mode 100644 index 00000000..7bdf019d --- /dev/null +++ b/IDR/resources/queries/bimber_data/idrAnchorDateSource.sql @@ -0,0 +1,21 @@ +SELECT + +Rh as subjectId, +PID0 as date, +'SIV Infection' as eventLabel, +'Hansen/IDR' as dataSource + +FROM bimber_data.subjects +WHERE PID0 IS NOT NULL + +UNION ALL + +SELECT + +Rh as subjectId, +D0 as date, +'Vaccination Start' as eventLabel, +'Hansen/IDR' as dataSource + +FROM bimber_data.subjects +WHERE D0 IS NOT NULL \ No newline at end of file diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml index dc9d527c..24901f48 100644 --- a/SivStudies/resources/etls/idr-data.xml +++ b/SivStudies/resources/etls/idr-data.xml @@ -133,6 +133,27 @@ + + + + + + + + + + + + + + + + + + + + + From 099ef814f50d7314236a46858e7a79c503a5105f Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 14:04:13 -0700 Subject: [PATCH 34/52] Correct field case --- .../queries/study/outcomes.query.xml | 4 --- .../queries/study/studyData.query.xml | 2 +- .../study/datasets/datasets_metadata.xml | 28 +++++++++---------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/SivStudies/resources/queries/study/outcomes.query.xml b/SivStudies/resources/queries/study/outcomes.query.xml index e655eb67..f036d859 100644 --- a/SivStudies/resources/queries/study/outcomes.query.xml +++ b/SivStudies/resources/queries/study/outcomes.query.xml @@ -20,10 +20,6 @@ Data Source - - Key - true - diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml index 8522b6c9..e824f95d 100644 --- a/SivStudies/resources/queries/study/studyData.query.xml +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -13,7 +13,7 @@ End Date - + Key true true diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index c45b36e2..0cbabcdf 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -13,7 +13,7 @@ varchar - + entityid true @@ -40,7 +40,7 @@ varchar - + entityid true @@ -103,7 +103,7 @@ varchar - + entityid true @@ -142,7 +142,7 @@ varchar - + entityid true @@ -166,7 +166,7 @@ varchar - + entityid true @@ -215,7 +215,7 @@ varchar - + entityid true @@ -256,7 +256,7 @@ varchar - + entityid false @@ -301,7 +301,7 @@ varchar - + entityid true @@ -340,7 +340,7 @@ varchar - + entityid true @@ -376,7 +376,7 @@ varchar - + entityid true @@ -411,7 +411,7 @@ varchar - + entityid true @@ -437,7 +437,7 @@ varchar - + entityid true @@ -463,7 +463,7 @@ varchar - + entityid true @@ -501,7 +501,7 @@ varchar - + entityid true From ba9b10697c7eea9f89b786d33cc391091a7372c3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 14:58:17 -0700 Subject: [PATCH 35/52] Set query config --- .../org/labkey/sivstudies/etl/SubjectScopedSelect.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index c3db2a86..8ad1133b 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -12,6 +12,7 @@ import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; import org.labkey.api.di.DataIntegrationService; import org.labkey.api.di.TaskRefTask; import org.labkey.api.pipeline.CancelledException; @@ -50,6 +51,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.labkey.api.gwt.client.AuditBehaviorType.NONE; + public class SubjectScopedSelect implements TaskRefTask { protected final Map _settings = new CaseInsensitiveHashMap<>(); @@ -167,7 +170,7 @@ private void processBatch(List subjects, Logger log, PipelineJob job) log.info("batch " + i); checkCancelled(job); - qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), batch, null, null); + qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), batch, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); } } else @@ -197,7 +200,7 @@ private void processBatch(List subjects, Logger log, PipelineJob job) checkCancelled(job); BatchValidationException bve = new BatchValidationException(); - qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), batch, bve, null, null); + qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), batch, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); if (bve.hasErrors()) { throw bve; @@ -233,7 +236,7 @@ else if (getMode() == MODE.UPDATE_ONLY) return map; }).toList(); - qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), batch, keys, bve, null, null); + qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), batch, keys, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); if (bve.hasErrors()) { throw bve; From dbad9a8c7becb6ba05da6e55ac33432b01420bd6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 16:59:36 -0700 Subject: [PATCH 36/52] Avoid invalid dates --- SivStudies/resources/etls/idr-data.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml index 24901f48..119bc475 100644 --- a/SivStudies/resources/etls/idr-data.xml +++ b/SivStudies/resources/etls/idr-data.xml @@ -17,6 +17,7 @@ + From 12734be14de9a5a2d5da1e371ef7cbb8556f5583 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 21:08:30 -0700 Subject: [PATCH 37/52] Create annotation to track UtilityActions and Admin Console page to list them --- .../sivstudies/etl/SubjectScopedSelect.java | 1 + .../query/ViralLoadsTriggerFactory.java | 6 +- primeseq/build.gradle | 1 + .../labkey/primeseq/PrimeseqController.java | 84 +------------------ 4 files changed, 9 insertions(+), 83 deletions(-) diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index 8ad1133b..2803b6d8 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -126,6 +126,7 @@ public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobExcepti private void checkCancelled(PipelineJob job) { + job.updateStatusForTask(); if (job.isCancelled()) { throw new CancelledException(); diff --git a/SivStudies/src/org/labkey/sivstudies/query/ViralLoadsTriggerFactory.java b/SivStudies/src/org/labkey/sivstudies/query/ViralLoadsTriggerFactory.java index 7f9bb267..3b4e7fea 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/ViralLoadsTriggerFactory.java +++ b/SivStudies/src/org/labkey/sivstudies/query/ViralLoadsTriggerFactory.java @@ -1,5 +1,6 @@ package org.labkey.sivstudies.query; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -14,6 +15,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; public class ViralLoadsTriggerFactory implements TriggerFactory { @@ -69,9 +71,9 @@ private void handlePVL(@Nullable Map row, Container c, Validatio { row.put("result", val); } - else if (val.startsWith("Below")) + else if (val.toLowerCase().startsWith("below")) { - val = val.replace("Below ", ""); + StringUtils.replaceIgnoreCase(val, "below ", ""); if (NumberUtils.isCreatable(val)) { row.put("result", val); diff --git a/primeseq/build.gradle b/primeseq/build.gradle index 05f2ed67..a4856250 100644 --- a/primeseq/build.gradle +++ b/primeseq/build.gradle @@ -12,6 +12,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:cluster", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:jbrowse", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index 3606da7e..bcd58996 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -33,12 +33,8 @@ import org.labkey.api.data.ContainerType; import org.labkey.api.data.DbScope; import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.discvrcore.annotation.UtilityAction; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipeRoot; @@ -47,13 +43,10 @@ import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineStatusFile; import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.RequiresSiteAdmin; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.HasJobParams; import org.labkey.api.sequenceanalysis.pipeline.JobResourceSettings; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; @@ -229,6 +222,7 @@ public URLHelper getSuccessURL(Object o) } } + @UtilityAction(label = "Update File Paths", description = "This action is designed to bulk update filepaths stored in the database, such as when a folder's file root is updated. This circumvents LabKey's normal file update listeners, and should only be performed if you are certain this is what you want. A reason for this is because the default codepath can be slow with extremely large moves.") @RequiresSiteAdmin public static class UpdateFilePathsAction extends ConfirmAction { @@ -631,6 +625,7 @@ public Collection getJobRowIds() } } + @UtilityAction(label = "Perform MHC Cleanup", description = "This will run a pipeline job to delete low-frequency MHC results to save space") @RequiresPermission(UpdatePermission.class) public static class PerformMhcCleanupAction extends ConfirmAction { @@ -803,77 +798,4 @@ public void setRestartJobs(boolean restartJobs) _restartJobs = restartJobs; } } - - @RequiresSiteAdmin - public static class FixSbtAction extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(Object o, BindException errors) throws Exception - { - setTitle("Fix SBT Errors"); - - return new HtmlView(HtmlString.of("This will update filepaths on SBT outputs. Do you want to continue?")); - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - new TableSelector(QueryService.get().getUserSchema(getUser(), getContainer(), "sequenceanalysis").getTable("outputfiles"), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("category"), "SBT Results"), null).forEachResults(rs -> { - SequenceOutputFile so = SequenceOutputFile.getForId(rs.getInt(FieldKey.fromString("rowid"))); - - File f = so.getFile(); - if (f.exists()) - { - return; - } - - ExpRun run = ExperimentService.get().getExpRun(so.getRunId()); - PipelineStatusFile sf = PipelineService.get().getStatusFile(run.getJobId()); - File logFile = new File(sf.getFilePath()); - File root = logFile.getParentFile(); - File [] dirs = root.listFiles(fn -> { - return fn.isDirectory() & !fn.getName().equalsIgnoreCase("Shared"); - }); - - if (dirs == null || dirs.length == 0) - { - _log.error("Unable to file directory for: " + f.getPath()); - return; - } - - File parent = new File(dirs[0], "Alignment"); - File [] children = parent.listFiles(fn -> { - return fn.getName().endsWith(".sbt_hits.txt.gz"); - }); - - if (children == null || children.length != 1) - { - _log.error("Unable to file child under: " + parent.getPath()); - return; - } - - _log.info("Found: " + children[0].getPath()); - - ExpData d = so.getExpData(); - d.setDataFileURI(children[0].toURI()); - - d.save(getUser()); - }); - - return true; - } - - @Override - public void validateCommand(Object o, Errors errors) - { - - } - - @NotNull - @Override - public URLHelper getSuccessURL(Object o) - { - return PageFlowUtil.urlProvider(PipelineUrls.class).urlBegin(getContainer()); - } - } } \ No newline at end of file From 9968f884138df0a28dc15e719a5059429dc20ba6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 Jun 2025 22:35:49 -0700 Subject: [PATCH 38/52] Add many default views --- .../study/additionalDatatypes.query.xml | 2 +- .../study/additionalDatatypes/.qview.xml | 12 +++ .../queries/study/assignment.query.xml | 2 +- .../queries/study/assignment/.qview.xml | 15 ++++ .../queries/study/demographics.query.xml | 6 +- .../queries/study/demographics/.qview.xml | 4 + .../resources/queries/study/flags.query.xml | 2 +- .../resources/queries/study/flags/.qview.xml | 12 +++ .../resources/queries/study/flow.query.xml | 5 +- .../resources/queries/study/flow/.qview.xml | 16 ++++ .../queries/study/genetics.query.xml | 2 +- .../queries/study/genetics/.qview.xml | 16 ++++ .../queries/study/immunizations.query.xml | 2 +- .../queries/study/immunizations/.qview.xml | 16 ++++ .../resources/queries/study/labwork.query.xml | 13 ++- .../queries/study/labwork/.qview.xml | 18 +++++ .../queries/study/outcomes.query.xml | 2 +- .../queries/study/outcomes/.qview.xml | 14 ++++ .../queries/study/procedures.query.xml | 4 +- .../queries/study/procedures/.qview.xml | 14 ++++ .../resources/queries/study/samples.query.xml | 2 +- .../queries/study/samples/.qview.xml | 17 ++++ .../queries/study/studyData.query.xml | 6 ++ .../queries/study/treatments.query.xml | 2 +- .../queries/study/treatments/.qview.xml | 18 +++++ .../queries/study/viralLoads/.qview.xml | 17 ++++ .../queries/study/viralloads.query.xml | 13 ++- .../resources/queries/study/weight.query.xml | 2 +- .../resources/queries/study/weight/.qview.xml | 14 ++++ .../query/SivStudiesCustomizer.java | 79 +++++++++++-------- 30 files changed, 296 insertions(+), 51 deletions(-) create mode 100644 SivStudies/resources/queries/study/additionalDatatypes/.qview.xml create mode 100644 SivStudies/resources/queries/study/assignment/.qview.xml create mode 100644 SivStudies/resources/queries/study/flags/.qview.xml create mode 100644 SivStudies/resources/queries/study/flow/.qview.xml create mode 100644 SivStudies/resources/queries/study/genetics/.qview.xml create mode 100644 SivStudies/resources/queries/study/immunizations/.qview.xml create mode 100644 SivStudies/resources/queries/study/labwork/.qview.xml create mode 100644 SivStudies/resources/queries/study/outcomes/.qview.xml create mode 100644 SivStudies/resources/queries/study/procedures/.qview.xml create mode 100644 SivStudies/resources/queries/study/samples/.qview.xml create mode 100644 SivStudies/resources/queries/study/treatments/.qview.xml create mode 100644 SivStudies/resources/queries/study/viralLoads/.qview.xml create mode 100644 SivStudies/resources/queries/study/weight/.qview.xml diff --git a/SivStudies/resources/queries/study/additionalDatatypes.query.xml b/SivStudies/resources/queries/study/additionalDatatypes.query.xml index d14427e2..07aff712 100644 --- a/SivStudies/resources/queries/study/additionalDatatypes.query.xml +++ b/SivStudies/resources/queries/study/additionalDatatypes.query.xml @@ -13,7 +13,7 @@ Category - Data Source + Data Source Description diff --git a/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml b/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml new file mode 100644 index 00000000..83423e55 --- /dev/null +++ b/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml @@ -0,0 +1,12 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml index 66e945ed..ed31de81 100644 --- a/SivStudies/resources/queries/study/assignment.query.xml +++ b/SivStudies/resources/queries/study/assignment.query.xml @@ -22,7 +22,7 @@ Cohort ID - Data Source + Data Source Category diff --git a/SivStudies/resources/queries/study/assignment/.qview.xml b/SivStudies/resources/queries/study/assignment/.qview.xml new file mode 100644 index 00000000..b10c793d --- /dev/null +++ b/SivStudies/resources/queries/study/assignment/.qview.xml @@ -0,0 +1,15 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographics.query.xml b/SivStudies/resources/queries/study/demographics.query.xml index d9d6914f..49f10a6d 100644 --- a/SivStudies/resources/queries/study/demographics.query.xml +++ b/SivStudies/resources/queries/study/demographics.query.xml @@ -51,16 +51,16 @@ Species - Mother + Mother/Dam - Father + Father/Sire Status - Data Source + Data Source diff --git a/SivStudies/resources/queries/study/demographics/.qview.xml b/SivStudies/resources/queries/study/demographics/.qview.xml index 081c2872..677d864d 100644 --- a/SivStudies/resources/queries/study/demographics/.qview.xml +++ b/SivStudies/resources/queries/study/demographics/.qview.xml @@ -12,6 +12,10 @@ + + + + diff --git a/SivStudies/resources/queries/study/flags.query.xml b/SivStudies/resources/queries/study/flags.query.xml index 91419bce..23071f2b 100644 --- a/SivStudies/resources/queries/study/flags.query.xml +++ b/SivStudies/resources/queries/study/flags.query.xml @@ -16,7 +16,7 @@ Flag - Data Source + Data Source diff --git a/SivStudies/resources/queries/study/flags/.qview.xml b/SivStudies/resources/queries/study/flags/.qview.xml new file mode 100644 index 00000000..91db6ed4 --- /dev/null +++ b/SivStudies/resources/queries/study/flags/.qview.xml @@ -0,0 +1,12 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/flow.query.xml b/SivStudies/resources/queries/study/flow.query.xml index 5f79cd06..63f49ed2 100644 --- a/SivStudies/resources/queries/study/flow.query.xml +++ b/SivStudies/resources/queries/study/flow.query.xml @@ -24,9 +24,12 @@ Units - + Comments + + Data Source + diff --git a/SivStudies/resources/queries/study/flow/.qview.xml b/SivStudies/resources/queries/study/flow/.qview.xml new file mode 100644 index 00000000..c00809cd --- /dev/null +++ b/SivStudies/resources/queries/study/flow/.qview.xml @@ -0,0 +1,16 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/genetics.query.xml b/SivStudies/resources/queries/study/genetics.query.xml index e0aac8e5..93794571 100644 --- a/SivStudies/resources/queries/study/genetics.query.xml +++ b/SivStudies/resources/queries/study/genetics.query.xml @@ -24,7 +24,7 @@ Score - Data Source + Data Source diff --git a/SivStudies/resources/queries/study/genetics/.qview.xml b/SivStudies/resources/queries/study/genetics/.qview.xml new file mode 100644 index 00000000..bb737b81 --- /dev/null +++ b/SivStudies/resources/queries/study/genetics/.qview.xml @@ -0,0 +1,16 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml index 688745a5..b6dd4c94 100644 --- a/SivStudies/resources/queries/study/immunizations.query.xml +++ b/SivStudies/resources/queries/study/immunizations.query.xml @@ -27,7 +27,7 @@ Reason - Data Source + Data Source diff --git a/SivStudies/resources/queries/study/immunizations/.qview.xml b/SivStudies/resources/queries/study/immunizations/.qview.xml new file mode 100644 index 00000000..8264aae3 --- /dev/null +++ b/SivStudies/resources/queries/study/immunizations/.qview.xml @@ -0,0 +1,16 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/labwork.query.xml b/SivStudies/resources/queries/study/labwork.query.xml index 4bbfcf11..c0808ad1 100644 --- a/SivStudies/resources/queries/study/labwork.query.xml +++ b/SivStudies/resources/queries/study/labwork.query.xml @@ -35,7 +35,18 @@ Method - Data Source + Data Source + + + Result OOR Indicator + true + + + true + + + Result In Range? + true diff --git a/SivStudies/resources/queries/study/labwork/.qview.xml b/SivStudies/resources/queries/study/labwork/.qview.xml new file mode 100644 index 00000000..55367247 --- /dev/null +++ b/SivStudies/resources/queries/study/labwork/.qview.xml @@ -0,0 +1,18 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/outcomes.query.xml b/SivStudies/resources/queries/study/outcomes.query.xml index f036d859..59e38fca 100644 --- a/SivStudies/resources/queries/study/outcomes.query.xml +++ b/SivStudies/resources/queries/study/outcomes.query.xml @@ -18,7 +18,7 @@ Comments - Data Source + Data Source diff --git a/SivStudies/resources/queries/study/outcomes/.qview.xml b/SivStudies/resources/queries/study/outcomes/.qview.xml new file mode 100644 index 00000000..546bccf3 --- /dev/null +++ b/SivStudies/resources/queries/study/outcomes/.qview.xml @@ -0,0 +1,14 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/procedures.query.xml b/SivStudies/resources/queries/study/procedures.query.xml index 6624d04d..06cd4bdc 100644 --- a/SivStudies/resources/queries/study/procedures.query.xml +++ b/SivStudies/resources/queries/study/procedures.query.xml @@ -7,7 +7,7 @@ - true + Category @@ -16,7 +16,7 @@ Procedure - Data Source + Data Source diff --git a/SivStudies/resources/queries/study/procedures/.qview.xml b/SivStudies/resources/queries/study/procedures/.qview.xml new file mode 100644 index 00000000..01c43bfd --- /dev/null +++ b/SivStudies/resources/queries/study/procedures/.qview.xml @@ -0,0 +1,14 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml index 922600f5..a65518ef 100644 --- a/SivStudies/resources/queries/study/samples.query.xml +++ b/SivStudies/resources/queries/study/samples.query.xml @@ -24,7 +24,7 @@ Quantity Units - Data Source + Data Source diff --git a/SivStudies/resources/queries/study/samples/.qview.xml b/SivStudies/resources/queries/study/samples/.qview.xml new file mode 100644 index 00000000..830bf244 --- /dev/null +++ b/SivStudies/resources/queries/study/samples/.qview.xml @@ -0,0 +1,17 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml index e824f95d..4a1d46d7 100644 --- a/SivStudies/resources/queries/study/studyData.query.xml +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -19,6 +19,12 @@ true false + + true + + + true +