From 578024e129e888f2f684207cda81eea1967b165e Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 23 Jul 2025 17:57:33 -0700
Subject: [PATCH 01/24] Update mGAP Query

---
 mGAP/resources/queries/mGAP/subjectsSource.query.xml | 1 +
 mGAP/resources/queries/mGAP/subjectsSource.sql       | 8 +++++++-
 mGAP/resources/views/mgapDataDashboard.html          | 2 +-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/mGAP/resources/queries/mGAP/subjectsSource.query.xml b/mGAP/resources/queries/mGAP/subjectsSource.query.xml
index 6185f7741..7d580fa2b 100644
--- a/mGAP/resources/queries/mGAP/subjectsSource.query.xml
+++ b/mGAP/resources/queries/mGAP/subjectsSource.query.xml
@@ -2,6 +2,7 @@
     <metadata>
         <tables xmlns="http://labkey.org/data/xml">
             <table tableName="" tableDbType="TABLE">
+                <tableTitle>mGAP Subject/Demographics Source Data</tableTitle>
                 <pkColumnName>subjectName</pkColumnName>
             </table>
         </tables>
diff --git a/mGAP/resources/queries/mGAP/subjectsSource.sql b/mGAP/resources/queries/mGAP/subjectsSource.sql
index 3a1ed394c..ce5a82461 100644
--- a/mGAP/resources/queries/mGAP/subjectsSource.sql
+++ b/mGAP/resources/queries/mGAP/subjectsSource.sql
@@ -10,9 +10,15 @@ SELECT
     WHEN s.Id IS NOT NULL THEN 'ONPRC'
     ELSE NULL END as center,
   d.status as status,
-  m.subjectname as originalId
+  m.subjectname as originalId,
+  p1.externalAlias as sire,
+  coalesce(s.Id.parents.sire, d.sire) as originalSire,
+  p2.externalAlias as dam,
+  coalesce(s.Id.parents.dam, d.dam) as originalDam,
 
 FROM mgap.animalMapping m
 LEFT JOIN "/Internal/PMR/".study.demographics s ON (m.subjectname = s.Id)
 LEFT JOIN mgap.demographics d ON (m.subjectname = d.subjectname)
+LEFT JOIN mgap.animalMapping p1 ON (p1.subjectname = coalesce(s.Id.parents.sire, d.sire))
+LEFT JOIN mgap.animalMapping p2 ON (p2.subjectname = coalesce(s.Id.parents.dam, d.dam))
 WHERE (s.Id IS NOT NULL OR d.subjectname IS NOT NULL)
\ No newline at end of file
diff --git a/mGAP/resources/views/mgapDataDashboard.html b/mGAP/resources/views/mgapDataDashboard.html
index 87cf98a9c..5894d4de3 100644
--- a/mGAP/resources/views/mgapDataDashboard.html
+++ b/mGAP/resources/views/mgapDataDashboard.html
@@ -100,7 +100,7 @@
                         name: 'Subject Information Synced to mGAP',
                         url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {
                             schemaName: 'mgap',
-                            queryName: 'subjectDatasetsSource'
+                            queryName: 'subjectsSource'
                         })
                     }]
                 },{

From f58e9f31777de0a89280274f554d580378ae3626 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sun, 27 Jul 2025 04:37:30 -0700
Subject: [PATCH 02/24] Expand study/cohort fields

---
 SivStudies/resources/queries/study/demographics/.qview.xml     | 2 +-
 .../resources/queries/study/demographics/Expanded.qview.xml    | 2 +-
 .../resources/queries/study/demographicsProjects.query.xml     | 3 +++
 SivStudies/resources/queries/study/demographicsProjects.sql    | 1 +
 4 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/SivStudies/resources/queries/study/demographics/.qview.xml b/SivStudies/resources/queries/study/demographics/.qview.xml
index 677d864d0..36ef67231 100644
--- a/SivStudies/resources/queries/study/demographics/.qview.xml
+++ b/SivStudies/resources/queries/study/demographics/.qview.xml
@@ -13,7 +13,7 @@
 <!--        </column>-->
         <column name="death"/>
         <column name="projects/allStudies"/>
-        <column name="projects/categories"/>
+        <column name="projects/subgroups"/>
         <column name="immunizations/immunizations"/>
         <column name="outcomes/outcomes"/>
     </columns>
diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml
index 8d3c3e733..1598334f0 100644
--- a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml
+++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml
@@ -7,7 +7,7 @@
         <column name="birth"/>
         <column name="death"/>
         <column name="projects/allStudies"/>
-        <column name="projects/categories"/>
+        <column name="projects/subgroups"/>
         <column name="immunizations/immunizations"/>
         <column name="sivART/allInfections"/>
         <column name="sivART/allART"/>
diff --git a/SivStudies/resources/queries/study/demographicsProjects.query.xml b/SivStudies/resources/queries/study/demographicsProjects.query.xml
index fc8f02e91..ec3dddc09 100644
--- a/SivStudies/resources/queries/study/demographicsProjects.query.xml
+++ b/SivStudies/resources/queries/study/demographicsProjects.query.xml
@@ -14,6 +14,9 @@
                     <column columnName="allStudies">
                         <columnTitle>All Studies</columnTitle>
                     </column>
+                    <column columnName="subgroups">
+                        <columnTitle>Subgroups/Treatments</columnTitle>
+                    </column>
                     <column columnName="rhCmvVaccines">
                         <columnTitle>RhCMV Vaccines?</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/demographicsProjects.sql b/SivStudies/resources/queries/study/demographicsProjects.sql
index e01f2a073..853db7d13 100644
--- a/SivStudies/resources/queries/study/demographicsProjects.sql
+++ b/SivStudies/resources/queries/study/demographicsProjects.sql
@@ -3,6 +3,7 @@ SELECT
   count(s.Id) as totalProjects,
   group_concat(DISTINCT s.study, char(10)) as allStudies,
   group_concat(DISTINCT s.category, char(10)) as categories,
+  group_concat(DISTINCT s.subgroup, char(10)) as subgroups,
 
   GROUP_CONCAT(distinct CASE WHEN s.category = 'RhCMV-Vaccines' THEN 'Yes' ELSE null END, char(10)) as rhCmvVaccines,
   GROUP_CONCAT(distinct CASE WHEN s.category = 'SIV/ART' THEN 'Yes' ELSE null END, char(10)) as sivArt

From 551ad881b03a1326a874b1f5079c12473729d2e8 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sun, 27 Jul 2025 04:54:49 -0700
Subject: [PATCH 03/24] Expand challenge query

---
 .../queries/study/demographicsChallengeAndArt.query.xml       | 3 +++
 .../resources/queries/study/demographicsChallengeAndArt.sql   | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml
index 8e3ec5353..135d018b6 100644
--- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml
+++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml
@@ -14,6 +14,9 @@
                     <column columnName="allART">
                         <columnTitle>ART</columnTitle>
                     </column>
+                    <column columnName="infectionDate">
+                        <columnTitle>Infection Date</columnTitle>
+                    </column>
                 </columns>
                 <titleColumn>allInfections</titleColumn>
             </table>
diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql
index ea898b425..6aebfaf74 100644
--- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql
+++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql
@@ -8,6 +8,10 @@ SELECT
       WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')')
       ELSE NULL
   END, char(10)) as allART,
+  min(CASE
+      WHEN t.category = 'SIV Infection' THEN t.date
+      ELSE NULL
+  END, char(10)) as infectionDate,
 
 FROM study.treatments t
 GROUP BY t.Id
\ No newline at end of file

From 14915d80219667adba76069590c4a4c48c1a59e0 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 29 Jul 2025 06:04:39 -0700
Subject: [PATCH 04/24] Fix challenge query

---
 .../resources/queries/study/demographicsChallengeAndArt.sql     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql
index 6aebfaf74..4e4f9fc73 100644
--- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql
+++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql
@@ -11,7 +11,7 @@ SELECT
   min(CASE
       WHEN t.category = 'SIV Infection' THEN t.date
       ELSE NULL
-  END, char(10)) as infectionDate,
+  END) as infectionDate,
 
 FROM study.treatments t
 GROUP BY t.Id
\ No newline at end of file

From ab29c30d07a2d0636fbf6427cc9ac1afe4735add Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 30 Jul 2025 10:00:09 -0700
Subject: [PATCH 05/24] Improve returnUrl logic for mGAP's login page

---
 mGAP/resources/views/login.html | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/mGAP/resources/views/login.html b/mGAP/resources/views/login.html
index 471911cf3..ead854e20 100644
--- a/mGAP/resources/views/login.html
+++ b/mGAP/resources/views/login.html
@@ -1,14 +1,30 @@
 <script type="text/javascript" nonce="<%=scriptNonce%>">
     (function($, LABKEY) {
         $(function(){
-            if (!LABKEY.Security.currentUser.isAdmin) {
-                var ctx = LABKEY.getModuleContext('mgap') || {};
-                var folder = ctx['MGAPContainer'];
-                if (folder) {
-                    var el = $('#returnUrl');
-                    el.attr('value', LABKEY.ActionURL.buildURL('project', 'start', folder));
+            // The intent of this block is to test whether the returnUrl is pointing to the same folder as the main mGAP project
+            // If true, always respect that.
+            // If the user is an admin, allow returnUrls to a different folder
+            // Otherwise, always point the user to the home page of the mGAP folder
+            var ctx = LABKEY.getModuleContext('mgap') || {};
+            var mgapFolder = ctx['MGAPContainer'];
+            if (mgapFolder) {
+                var returnUrl = LABKEY.ActionURL.getParameter('returnUrl')
+                if (returnUrl) {
+                    returnUrl = decodeURIComponent(returnUrl)
+
+                    var containerPath = LABKEY.ActionURL.getContainerName(returnUrl).toLowerCase()
+                    mgapFolder = mgapFolder.replaceAll('/', '').toLowerCase()
+                    if (LABKEY.Security.currentUser.isAdmin || containerPath === mgapFolder) {
+                        $('#returnUrl').attr('value', returnUrl);
+                    } else {
+                        $('#returnUrl').attr('value', LABKEY.ActionURL.buildURL('project', 'start', mgapFolder));
+                    }
                 }
             }
+            else {
+                // Ignore the returnUrl in this case, since we cant tell whether it's from the expected project:
+                $('#returnUrl').attr('value', LABKEY.ActionURL.buildURL('project', 'start', 'home'));
+            }
         });
     }(jQuery, LABKEY));
 

From 2c1ec2217fcc59c00396549807b716183fcb3253 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 4 Aug 2025 06:43:51 -0700
Subject: [PATCH 06/24] Test fix and support VCF 4.4

---
 .../test/tests/external/labModules/GenotypeAssaysTest.java       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/GenotypeAssays/test/src/org/labkey/test/tests/external/labModules/GenotypeAssaysTest.java b/GenotypeAssays/test/src/org/labkey/test/tests/external/labModules/GenotypeAssaysTest.java
index ffc0aeac9..771c4e5bc 100644
--- a/GenotypeAssays/test/src/org/labkey/test/tests/external/labModules/GenotypeAssaysTest.java
+++ b/GenotypeAssays/test/src/org/labkey/test/tests/external/labModules/GenotypeAssaysTest.java
@@ -182,6 +182,7 @@ private void beadStudioImportTest()
 
         Ext4FieldRef.getForLabel(this, "Run Description").setValue("Description");
 
+        waitForElement(Ext4Helper.Locators.ext4Button("Download Example Data"));
         File exampleData = clickAndWaitForDownload(Ext4Helper.Locators.ext4Button("Download Example Data"));
 
         waitAndClick(Ext4Helper.Locators.radiobutton(this, "File Upload"));

From feff0ec558681f139150e28f04dc9c924fe8b010 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 4 Aug 2025 09:42:20 -0700
Subject: [PATCH 07/24] Ensure lifted VCF is sorted and indexed

---
 .../org/labkey/mgap/pipeline/AnnotationStep.java    | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java b/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java
index 95f1eb4f4..9dddabd9f 100644
--- a/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java
+++ b/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java
@@ -27,6 +27,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider;
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
 import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep;
 import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStepOutputImpl;
@@ -330,6 +331,18 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
         {
             LiftoverBcfToolsWrapper liftoverVcfRunner = new LiftoverBcfToolsWrapper(getPipelineCtx().getLogger());
             liftoverVcfRunner.doLiftover(currentVcf, chainFile, genome.getWorkingFastaFile(), grch37Genome.getWorkingFastaFile(), liftoverRejects, liftedToGRCh37);
+
+            // NOTE: the resulting VCF is not necessarily in chromosome sort order:
+            SequencePipelineService.get().sortVcf(liftedToGRCh37, null, genome.getSequenceDictionary(), getPipelineCtx().getLogger());
+
+            try
+            {
+                SequenceAnalysisService.get().ensureVcfIndex(liftedToGRCh37, getPipelineCtx().getLogger());
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
         }
         else
         {

From 1f2da5bdf9864affdbd9ebdfe07089603ba6db3b Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 4 Aug 2025 13:58:59 -0700
Subject: [PATCH 08/24] Ignore half-called GTs in KING

---
 mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java b/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java
index 9dddabd9f..63b49866e 100644
--- a/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java
+++ b/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java
@@ -1,6 +1,7 @@
 package org.labkey.mgap.pipeline;
 
 import htsjdk.samtools.util.Interval;
+import htsjdk.samtools.util.SequenceUtil;
 import htsjdk.variant.vcf.VCFFileReader;
 import org.apache.commons.lang3.StringUtils;
 import org.jetbrains.annotations.Nullable;
@@ -332,11 +333,11 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
             LiftoverBcfToolsWrapper liftoverVcfRunner = new LiftoverBcfToolsWrapper(getPipelineCtx().getLogger());
             liftoverVcfRunner.doLiftover(currentVcf, chainFile, genome.getWorkingFastaFile(), grch37Genome.getWorkingFastaFile(), liftoverRejects, liftedToGRCh37);
 
-            // NOTE: the resulting VCF is not necessarily in chromosome sort order:
-            SequencePipelineService.get().sortVcf(liftedToGRCh37, null, genome.getSequenceDictionary(), getPipelineCtx().getLogger());
-
             try
             {
+                // NOTE: the resulting VCF is not necessarily in chromosome sort order:
+                SequencePipelineService.get().sortROD(liftedToGRCh37, getPipelineCtx().getLogger(), 2);
+
                 SequenceAnalysisService.get().ensureVcfIndex(liftedToGRCh37, getPipelineCtx().getLogger());
             }
             catch (IOException e)

From 5b4d5fe9e5c6745511eefa6c50111c512058d19b Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 4 Aug 2025 14:44:14 -0700
Subject: [PATCH 09/24] Add pvl_outcomes table

---
 SivStudies/resources/data/reports.tsv         |  3 +-
 .../study/duplicateInfectionDates.query.xml   | 12 +++++++
 .../queries/study/duplicateInfectionDates.sql | 10 ++++++
 .../infectionAnchorDateDiscordance.query.xml  | 17 ++++++++++
 .../study/infectionAnchorDateDiscordance.sql  | 13 ++++++++
 .../queries/study/pvl_outcomes.query.xml      | 30 +++++++++++++++++
 .../queries/study/pvl_outcomes/.qview.xml     | 15 +++++++++
 .../study/datasets/datasets_manifest.xml      |  3 ++
 .../study/datasets/datasets_metadata.xml      | 32 +++++++++++++++++++
 9 files changed, 134 insertions(+), 1 deletion(-)
 create mode 100644 SivStudies/resources/queries/study/duplicateInfectionDates.query.xml
 create mode 100644 SivStudies/resources/queries/study/duplicateInfectionDates.sql
 create mode 100644 SivStudies/resources/queries/study/infectionAnchorDateDiscordance.query.xml
 create mode 100644 SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql
 create mode 100644 SivStudies/resources/queries/study/pvl_outcomes.query.xml
 create mode 100644 SivStudies/resources/queries/study/pvl_outcomes/.qview.xml

diff --git a/SivStudies/resources/data/reports.tsv b/SivStudies/resources/data/reports.tsv
index 65f6076a1..c9ba1c207 100644
--- a/SivStudies/resources/data/reports.tsv
+++ b/SivStudies/resources/data/reports.tsv
@@ -7,7 +7,8 @@ query	Clinical	study	hematologyPivot		CBC/Hematology
 query	Study Design	study	assignment		Project/Study Assignment	
 query	Research	study	flow		Flow Cytometry	
 query	Research	study	outcomes		Study Outcomes/Phenotypes	
-query	Research	study	genetics		Genetic Data / MHC	
+query	Research	study	pvl_outcomes		Study PVL-Based Outcomes
+query	Research	study	genetics		Genetic Data / MHC
 query	Clinical	study	procedures		Procedures	
 query	Research	study	viralLoads		Viral Loads	
 query	General	study	samples		Samples	
diff --git a/SivStudies/resources/queries/study/duplicateInfectionDates.query.xml b/SivStudies/resources/queries/study/duplicateInfectionDates.query.xml
new file mode 100644
index 000000000..8ea99a2c4
--- /dev/null
+++ b/SivStudies/resources/queries/study/duplicateInfectionDates.query.xml
@@ -0,0 +1,12 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="duplicateInfectionDates" tableDbType="NOT_IN_DB">
+                <tableTitle>Duplicate Infection Dates</tableTitle>
+                <columns>
+
+                </columns>
+            </table>
+        </tables>
+    </metadata>
+</query>
diff --git a/SivStudies/resources/queries/study/duplicateInfectionDates.sql b/SivStudies/resources/queries/study/duplicateInfectionDates.sql
new file mode 100644
index 000000000..a9f1a8acd
--- /dev/null
+++ b/SivStudies/resources/queries/study/duplicateInfectionDates.sql
@@ -0,0 +1,10 @@
+SELECT
+t.Id,
+t.date,
+'SIV Infection' as category
+
+FROM study.treatments t
+WHERE t.category = 'SIV Infection'
+
+GROUP BY t.Id, t.date
+HAVING count(*) > 1
\ No newline at end of file
diff --git a/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.query.xml b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.query.xml
new file mode 100644
index 000000000..2d5c8bab7
--- /dev/null
+++ b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.query.xml
@@ -0,0 +1,17 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="infectionAnchorDateDiscordance" tableDbType="NOT_IN_DB">
+                <tableTitle>Discordant Infection Dates</tableTitle>
+                <columns>
+                    <column columnName="treatmentTableDate">
+                        <columnTitle>Date (Treatments Table)</columnTitle>
+                    </column>
+                    <column columnName="subjectAnchorDatesTableDate">
+                        <columnTitle>Date (Anchor Dates Table)</columnTitle>
+                    </column>
+                </columns>
+            </table>
+        </tables>
+    </metadata>
+</query>
diff --git a/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql
new file mode 100644
index 000000000..13eabe059
--- /dev/null
+++ b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql
@@ -0,0 +1,13 @@
+SELECT
+    t.Id,
+    'SIV Infection' as category,
+    t.date as treatmentTableDate,
+    t2.date as subjectAnchorDatesTableDate
+
+FROM study.treatments t
+FULL JOIN studies.subjectAnchorDates t2 ON (
+    t2.subjectId = t.Id AND
+    t2.eventLabel = 'SIV Infection'
+)
+
+WHERE t.category = 'SIV Infection' AND t2.date != t.date
\ No newline at end of file
diff --git a/SivStudies/resources/queries/study/pvl_outcomes.query.xml b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
new file mode 100644
index 000000000..e06038802
--- /dev/null
+++ b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
@@ -0,0 +1,30 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="pvl_outcomes" tableDbType="TABLE" useColumnOrder="true">
+                <columns>
+                    <column columnName="Id"/>
+                    <column columnName="date">
+                        <columnTitle>Date</columnTitle>
+                        <formatString>Date</formatString>
+                    </column>
+                    <column columnName="outcome">
+                        <columnTitle>Outcome/Phenotype</columnTitle>
+                    </column>
+                    <column columnName="numeric_value">
+                        <columnTitle>Numeric Value</columnTitle>
+                    </column>
+                    <column columnName="string_value">
+                        <columnTitle>String Value</columnTitle>
+                    </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                    </column>
+                    <column columnName="dataSource">
+                        <columnTitle>Data Source</columnTitle>
+                    </column>
+                </columns>
+            </table>
+        </tables>
+    </metadata>
+</query>
diff --git a/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml b/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml
new file mode 100644
index 000000000..05ad5dcfa
--- /dev/null
+++ b/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml
@@ -0,0 +1,15 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView" hidden="false" canOverride="false">
+    <columns>
+        <column name="Id"/>
+        <column name="date"/>
+        <column name="outcome"/>
+        <column name="numeric_value"/>
+        <column name="string_value"/>
+        <column name="comments"/>
+        <column name="dataSource"/>
+    </columns>
+    <sorts>
+        <sort column="Id" descending="false"/>
+        <sort column="outcome" descending="true"/>
+    </sorts>
+</customView>
\ No newline at end of file
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml
index 715134ec4..12d37227d 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml
@@ -46,5 +46,8 @@
         <dataset name="outcomes" showByDefault="true" category="Research" type="Standard" id="15">
             <tags/>
         </dataset>
+        <dataset name="pvl_outcomes" showByDefault="true" category="Research" type="Standard" id="16">
+            <tags/>
+        </dataset>
     </datasets>
 </datasets>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 47f66507d..5eb90a75a 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -517,4 +517,36 @@
         </columns>
         <tableTitle>Outcomes/Phenotypes</tableTitle>
     </table>
+    <table tableName="pvl_outcomes" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <conceptURI>http://cpas.labkey.com/Study#ParticipantId</conceptURI>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <conceptURI>http://cpas.labkey.com/laboratory#sampleDate</conceptURI>
+            </column>
+            <column columnName="dataSource">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="objectId">
+                <datatype>entityid</datatype>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="outcome">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="numeric_value">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="string_value">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+        <tableTitle>PVL Outcomes</tableTitle>
+    </table>
 </tables>

From db6458f630ceaef24ad48d11a49ee59e59545b1c Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 5 Aug 2025 10:48:42 -0700
Subject: [PATCH 10/24] Use lsid instead of objectId for ETL

---
 mcc/resources/etls/wnprc-datasets.xml | 43 ++-------------------------
 1 file changed, 2 insertions(+), 41 deletions(-)

diff --git a/mcc/resources/etls/wnprc-datasets.xml b/mcc/resources/etls/wnprc-datasets.xml
index 97ecaf334..2fe8aa7b1 100644
--- a/mcc/resources/etls/wnprc-datasets.xml
+++ b/mcc/resources/etls/wnprc-datasets.xml
@@ -37,28 +37,6 @@
             </destination>
         </transform>
 
-        <!--<transform type="RemoteQueryTransformStep" id="birth">-->
-        <!--<description>Copy to target</description>-->
-        <!--<source remoteSource="WNPRC" schemaName="study" queryName="birth">-->
-        <!--<sourceColumns>-->
-        <!--<column>Id</column>-->
-        <!--<column>date</column>-->
-        <!--<column>gender</column>-->
-        <!--<column>species</column>-->
-        <!--<column>geographic_origin</column>-->
-        <!--<column>dam</column>-->
-        <!--<column>sire</column>-->
-        <!--<column>objectid</column>-->
-        <!--</sourceColumns>-->
-        <!--</source>-->
-        <!--<destination schemaName="study" queryName="birth" targetOption="merge" bulkLoad="true">-->
-        <!--<alternateKeys>-->
-        <!--<column name="objectid"/>-->
-        <!--</alternateKeys>-->
-        <!--</destination>-->
-
-        <!--</transform>-->
-
         <transform type="RemoteQueryTransformStep" id="weight">
             <description>Copy to target</description>
             <source remoteSource="WNPRC" schemaName="study" queryName="weight">
@@ -84,7 +62,7 @@
                     <column>date</column>
                     <column>enddate</column>
                     <column>title</column>
-                    <column>objectid</column>
+                    <column>lsid</column>
                 </sourceColumns>
             </source>
             <destination schemaName="study" queryName="flags" targetOption="merge" bulkLoad="true">
@@ -92,7 +70,7 @@
                     <column source="title" target="flag" transformClass="org.labkey.mcc.etl.ProjectAssignmentTransform"/>
                 </columnTransforms>
                 <alternateKeys>
-                    <column name="objectid"/>
+                    <column name="lsid"/>
                 </alternateKeys>
             </destination>
         </transform>
@@ -123,23 +101,6 @@
             </taskref>
         </transform>
 
-        <!--<transform type="RemoteQueryTransformStep" id="deaths">-->
-        <!--<description>Copy to target</description>-->
-        <!--<source remoteSource="WNPRC" schemaName="study" queryName="deaths">-->
-        <!--<sourceColumns>-->
-        <!--<column>Id</column>-->
-        <!--<column>date</column>-->
-        <!--<column>cause</column>-->
-        <!--<column>objectid</column>-->
-        <!--</sourceColumns>-->
-        <!--</source>-->
-        <!--<destination schemaName="study" queryName="deaths" targetOption="merge" bulkLoad="true">-->
-        <!--<alternateKeys>-->
-        <!--<column name="objectid"/>-->
-        <!--</alternateKeys>-->
-        <!--</destination>-->
-        <!--</transform>-->
-
         <!--This will create MCC aliases as-needed-->
         <transform id="step0" type="TaskRefTransformStep">
             <taskref ref="org.labkey.mcc.etl.PopulateIdsStep">

From 9f86bac0fed6f71144437f470a13e1de9e349968 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 6 Aug 2025 09:11:36 -0700
Subject: [PATCH 11/24] Use Audit table for PMR ETL deletes

---
 PMR/resources/etls/pmr-datasets.xml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml
index a0fc89c53..9bae7210c 100644
--- a/PMR/resources/etls/pmr-datasets.xml
+++ b/PMR/resources/etls/pmr-datasets.xml
@@ -148,7 +148,7 @@
                     <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
                 </sourceFilters>
             </source>
-            <destination schemaName="study" queryName="parentage" targetOption="truncate" bulkLoad="true" batchSize="2500">
+            <destination schemaName="study" queryName="parentage" targetOption="merge" bulkLoad="true" batchSize="2500">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -180,7 +180,7 @@
                     <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
                 </sourceFilters>
             </source>
-            <destination schemaName="study" queryName="birth" targetOption="truncate" bulkLoad="true" batchSize="2500">
+            <destination schemaName="study" queryName="birth" targetOption="merge" bulkLoad="true" batchSize="2500">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -188,9 +188,9 @@
         </transform>
     </transforms>
 
-<!--    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >-->
-<!--        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid"/>-->
-<!--    </incrementalFilter>-->
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid"/>
+    </incrementalFilter>
     <schedule>
         <!--11PM-->
         <cron expression="0 0 23 * * ?"/>

From d79c6a7830bf4c74e7ce88e09bd441ad6aa5fcf3 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 6 Aug 2025 12:23:01 -0700
Subject: [PATCH 12/24] Add celltype column to tcr stats table

---
 .../schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql         | 1 +
 .../schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql          | 1 +
 tcrdb/resources/schemas/tcrdb.xml                              | 3 +++
 tcrdb/src/org/labkey/tcrdb/TCRdbModule.java                    | 2 +-
 4 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql
 create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql

diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql
new file mode 100644
index 000000000..141403f5f
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql
@@ -0,0 +1 @@
+ALTER TABLE tcrdb.repertoire_stats ADD COLUMN celltype varchar(1000);
\ No newline at end of file
diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql
new file mode 100644
index 000000000..fe33f991d
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql
@@ -0,0 +1 @@
+ALTER TABLE tcrdb.repertoire_stats ADD celltype varchar(1000);
\ No newline at end of file
diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml
index b63843443..f152dc569 100644
--- a/tcrdb/resources/schemas/tcrdb.xml
+++ b/tcrdb/resources/schemas/tcrdb.xml
@@ -368,6 +368,9 @@
                     <fkColumnName>rowid</fkColumnName>
                 </fk>
             </column>
+            <column columnName="celltype">
+                <columnTitle>Cell Type</columnTitle>
+            </column>
             <column columnName="metricName">
                 <columnTitle>Metric Name</columnTitle>
             </column>
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
index 1f4726263..26089978d 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
@@ -46,7 +46,7 @@ public String getName()
     @Override
     public Double getSchemaVersion()
     {
-        return 15.58;
+        return 15.59;
     }
 
     @Override

From 22a436959e518602adc2911aeb21bf5483425034 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 7 Aug 2025 10:35:17 -0700
Subject: [PATCH 13/24] Add tcrdb column

---
 .../schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql         | 1 +
 .../schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql          | 1 +
 tcrdb/resources/schemas/tcrdb.xml                              | 3 +++
 tcrdb/src/org/labkey/tcrdb/TCRdbModule.java                    | 2 +-
 4 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql
 create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql

diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql
new file mode 100644
index 000000000..2a024139c
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql
@@ -0,0 +1 @@
+ALTER TABLE tcrdb.repertoire_stats ADD COLUMN samplesize int;
\ No newline at end of file
diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql
new file mode 100644
index 000000000..1300ca0e0
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql
@@ -0,0 +1 @@
+ALTER TABLE tcrdb.repertoire_stats ADD samplesize int;
\ No newline at end of file
diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml
index f152dc569..a8a486ce0 100644
--- a/tcrdb/resources/schemas/tcrdb.xml
+++ b/tcrdb/resources/schemas/tcrdb.xml
@@ -382,6 +382,9 @@
             </column>
             <column columnName="comment">
 
+            </column>
+            <column columnName="sampleSize">
+                <columnTitle>Sample Size</columnTitle>
             </column>
             <column columnName="container">
                 <isHidden>true</isHidden>
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
index 26089978d..39fca6481 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
@@ -46,7 +46,7 @@ public String getName()
     @Override
     public Double getSchemaVersion()
     {
-        return 15.59;
+        return 15.60;
     }
 
     @Override

From c04c297f9e7ea45a9534c531802164433d720983 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 7 Aug 2025 17:42:45 -0700
Subject: [PATCH 14/24] Add study validation notification

---
 .../study/infectionAnchorDateDiscordance.sql  |   5 +-
 .../labkey/sivstudies/SivStudiesModule.java   |   3 +
 .../SivStudiesDataValidationNotification.java | 125 ++++++++++++++++++
 .../query/SivStudiesCustomizer.java           |   7 +
 4 files changed, 139 insertions(+), 1 deletion(-)
 create mode 100644 SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java

diff --git a/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql
index 13eabe059..c19cd0943 100644
--- a/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql
+++ b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql
@@ -6,8 +6,11 @@ SELECT
 
 FROM study.treatments t
 FULL JOIN studies.subjectAnchorDates t2 ON (
+    t.category = 'SIV Infection' AND
     t2.subjectId = t.Id AND
     t2.eventLabel = 'SIV Infection'
 )
 
-WHERE t.category = 'SIV Infection' AND t2.date != t.date
\ No newline at end of file
+WHERE t2.date != t.date OR
+    t2.date IS NULL OR
+    t.date IS NULL
\ No newline at end of file
diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java
index 4bedeb33e..72d8b6e83 100644
--- a/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java
+++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java
@@ -20,8 +20,10 @@
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.Container;
 import org.labkey.api.ldk.ExtendedSimpleModule;
+import org.labkey.api.ldk.notification.NotificationService;
 import org.labkey.api.module.ModuleContext;
 import org.labkey.api.studies.StudiesService;
+import org.labkey.sivstudies.notification.SivStudiesDataValidationNotification;
 import org.labkey.sivstudies.study.ArtInitiationEventProvider;
 import org.labkey.sivstudies.study.SivInfectionEventProvider;
 
@@ -56,6 +58,7 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext)
     {
         StudiesService.get().registerEventProvider(new SivInfectionEventProvider());
         StudiesService.get().registerEventProvider(new ArtInitiationEventProvider());
+        NotificationService.get().registerNotification(new SivStudiesDataValidationNotification());
     }
 
     @Override
diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
new file mode 100644
index 000000000..314c662a0
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
@@ -0,0 +1,125 @@
+package org.labkey.sivstudies.notification;
+
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.ldk.notification.AbstractNotification;
+import org.labkey.api.module.ModuleLoader;
+import org.labkey.api.query.QueryService;
+import org.labkey.api.query.UserSchema;
+import org.labkey.api.security.User;
+import org.labkey.sivstudies.SivStudiesModule;
+
+import java.util.Date;
+
+public class SivStudiesDataValidationNotification extends AbstractNotification
+{
+    public SivStudiesDataValidationNotification()
+    {
+        super(ModuleLoader.getInstance().getModule(SivStudiesModule.class));
+    }
+
+    @Override
+    public String getName()
+    {
+        return "SIV Studies Validation";
+    }
+
+    @Override
+    public String getCategory()
+    {
+        return "Studies";
+    }
+
+    @Override
+    public String getCronString()
+    {
+        return "0 0 6 * * ?";
+    }
+
+    @Override
+    public String getScheduleDescription()
+    {
+        return "every day at 6:00AM";
+    }
+
+    @Override
+    public String getDescription()
+    {
+        return "SIV Study Data Validation Alerts";
+    }
+
+    @Override
+    public String getEmailSubject(Container c)
+    {
+        return "SIV Study Data Validation: " + c.getName();
+    }
+
+    @Override
+    public @Nullable String getMessageBodyHTML(Container c, User u)
+    {
+        StringBuilder msg = new StringBuilder();
+        Date now = new Date();
+
+        duplicateInfectionCheck(c, u, msg);
+
+        if (!msg.isEmpty())
+        {
+            msg.insert(0, "This email contains a series of automatic alerts about the SIV study data.  It was run on: " + getDateFormat(c).format(now) + " at " + AbstractNotification._timeFormat.format(now) + ".<p>");
+        }
+
+        return msg.toString();
+    }
+
+    private TableInfo getTableInfo(User u, Container c, String schemaName, String queryName)
+    {
+        UserSchema us = QueryService.get().getUserSchema(u, c, schemaName);
+        if (us == null)
+        {
+            throw new IllegalStateException("Missing user schema: " + schemaName);
+        }
+
+        TableInfo ti = us.getTable(queryName);
+        if (ti == null)
+        {
+            throw new IllegalStateException("Missing table: " + schemaName + "." + queryName);
+        }
+
+        return ti;
+    }
+
+    private void duplicateInfectionCheck(Container c, User u, StringBuilder msg)
+    {
+        String schemaName = "study";
+        String queryName = "duplicateInfectionDates";
+
+        TableInfo ti = getTableInfo(u, c, schemaName, queryName);
+
+        TableSelector ts = new TableSelector(ti);
+        long count = ts.getRowCount();
+        if (count > 0)
+        {
+            msg.append("<b>WARNING: There are ").append(count).append(" duplicate infection date records</b><br>\n");
+            msg.append("<p><a href='").append(getExecuteQueryUrl(c, schemaName, queryName, null)).append("'>Click here to view them</a><br>\n\n");
+            msg.append("<hr>\n\n");
+        }
+    }
+
+    private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder msg)
+    {
+        String schemaName = "study";
+        String queryName = "infectionAnchorDateDiscordance";
+
+        TableInfo ti = getTableInfo(u, c, schemaName, queryName);
+
+        TableSelector ts = new TableSelector(ti);
+        long count = ts.getRowCount();
+        if (count > 0)
+        {
+            msg.append("<b>WARNING: There are ").append(count).append(" records with discordant treatment and anchor date SIV infection records</b><br>\n");
+            msg.append("<p><a href='").append(getExecuteQueryUrl(c, schemaName, queryName, null)).append("'>Click here to view them</a><br>\n\n");
+            msg.append("<hr>\n\n");
+        }
+    }
+}
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index ebaeb3322..9d82243ac 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -293,6 +293,13 @@ private void appendPvlColumns(DatasetTable ds, String subjectColName, String dat
             newColOOR.setHidden(true);
 
             ti.addColumn(newColOOR);
+
+            String nameHasPvl = name + "HasPvl";
+            SQLFragment sqlHasPvl = new SQLFragment("(SELECT CASE WHEN count(*) > 0 THEN " + ti.getSqlDialect().getBooleanTRUE() + " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END as expr FROM studydataset." + tableName + " t WHERE t.participantid = " + ExprColumn.STR_TABLE_ALIAS + ".participantid AND t.sampletype = 'Plasma' AND t.target = 'SIV')");
+            ExprColumn newColHasPvl = new ExprColumn(ti, nameHasPvl, sqlHasPvl, JdbcType.DOUBLE, subjectCol, dateCol);
+            newColHasPvl.setDescription("Displays whether this animal has any positive SIV PVLs");
+            newColHasPvl.setLabel("Has Positive SIV PVL?");
+            ti.addColumn(newColHasPvl);
         }
     }
 

From 22c203907314c5fdc0af52ebc28df57076501f42 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 8 Aug 2025 06:18:34 -0700
Subject: [PATCH 15/24] Correct column name in ETL

---
 PMR/resources/etls/pmr-datasets.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml
index 9bae7210c..2b13a3f27 100644
--- a/PMR/resources/etls/pmr-datasets.xml
+++ b/PMR/resources/etls/pmr-datasets.xml
@@ -188,8 +188,8 @@
         </transform>
     </transforms>
 
-    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >
-        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid"/>
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="Created" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid" timestampColumnName="Created"/>
     </incrementalFilter>
     <schedule>
         <!--11PM-->

From 0e99631526ed99eec36ba5855436e31b0116f667 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 8 Aug 2025 07:05:44 -0700
Subject: [PATCH 16/24] Add ETL task to verify source/dest row counts

---
 PMR/build.gradle                              |   1 +
 PMR/resources/etls/pmr-datasets.xml           |  28 ++
 PMR/resources/etls/pmr-demographics.xml       |  18 +-
 SivStudies/resources/views/studiesAdmin.html  |   3 +
 .../labkey/primeseq/etl/VerifyRowCount.java   | 264 ++++++++++++++++++
 5 files changed, 310 insertions(+), 4 deletions(-)
 create mode 100644 primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java

diff --git a/PMR/build.gradle b/PMR/build.gradle
index dccb30067..f33b6455c 100644
--- a/PMR/build.gradle
+++ b/PMR/build.gradle
@@ -9,6 +9,7 @@ dependencies {
     BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:premiumModules:dataintegration", depProjectConfig: "published", depExtension: "module")
     BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module")
     BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:onprcEHRModules:GeneticsCore", depProjectConfig: "published", depExtension: "module")
+    BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:BimberLabKeyModules:primeseq", depProjectConfig: "published", depExtension: "module")
 
     BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "apiJarFile")
     BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:premiumModules:dataintegration", depProjectConfig: "apiJarFile")
diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml
index 2b13a3f27..ebb5aa1de 100644
--- a/PMR/resources/etls/pmr-datasets.xml
+++ b/PMR/resources/etls/pmr-datasets.xml
@@ -154,6 +154,20 @@
                 </alternateKeys>
             </destination>
         </transform>
+        <transform id="dataset9_rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="parentage"/>
+                    <setting name="sourceColumn" value="objectId"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="parentage"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
         <transform id="dataset10" type="RemoteQueryTransformStep">
             <source schemaName="study" queryName="birth" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
                 <sourceColumns>
@@ -186,6 +200,20 @@
                 </alternateKeys>
             </destination>
         </transform>
+        <transform id="dataset10_rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="birth"/>
+                    <setting name="sourceColumn" value="objectId"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="birth"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
     </transforms>
 
     <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="Created" >
diff --git a/PMR/resources/etls/pmr-demographics.xml b/PMR/resources/etls/pmr-demographics.xml
index 03b2bf94e..883f477ce 100644
--- a/PMR/resources/etls/pmr-demographics.xml
+++ b/PMR/resources/etls/pmr-demographics.xml
@@ -30,12 +30,22 @@
                 </alternateKeys>
             </destination>
         </transform>
+        <transform id="dataset9_rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="demographics"/>
+                    <setting name="sourceColumn" value="objectId"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="demographics"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
     </transforms>
 
-<!--    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >-->
-<!--        &lt;!&ndash; TODO: enable once PRIMe deploys 23.7 &ndash;&gt;-->
-<!--        &lt;!&ndash;<deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid"/>&ndash;&gt;-->
-<!--    </incrementalFilter>-->
     <schedule>
         <!--11PM-->
         <cron expression="0 0 23 * * ?"/>
diff --git a/SivStudies/resources/views/studiesAdmin.html b/SivStudies/resources/views/studiesAdmin.html
index b17398a9c..fb5e91025 100644
--- a/SivStudies/resources/views/studiesAdmin.html
+++ b/SivStudies/resources/views/studiesAdmin.html
@@ -24,6 +24,9 @@
                 items: [{
                     name: 'Import/Re-import Study and Reference Data',
                     url: LABKEY.ActionURL.buildURL('sivstudies', 'importStudy.view')
+                },{
+                    name: 'Notification Admin',
+                    url: LABKEY.ActionURL.buildURL('ldk', 'notificationAdmin.view')
                 }]
             }]
         }]
diff --git a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
new file mode 100644
index 000000000..331c37dea
--- /dev/null
+++ b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
@@ -0,0 +1,264 @@
+package org.labkey.primeseq.etl;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
+import org.labkey.api.collections.CaseInsensitiveHashMap;
+import org.labkey.api.data.CompareType;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.SimpleFilter;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.di.DataIntegrationService;
+import org.labkey.api.di.TaskRefTask;
+import org.labkey.api.pipeline.PipelineJob;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.pipeline.RecordedActionSet;
+import org.labkey.api.query.QueryService;
+import org.labkey.api.query.SimpleValidationError;
+import org.labkey.api.query.UserSchema;
+import org.labkey.api.query.ValidationError;
+import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.view.ActionURL;
+import org.labkey.api.writer.ContainerUser;
+import org.labkey.remoteapi.CommandException;
+import org.labkey.remoteapi.query.Filter;
+import org.labkey.remoteapi.query.SelectRowsCommand;
+import org.labkey.remoteapi.query.SelectRowsResponse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class VerifyRowCount implements TaskRefTask
+{
+    protected final Map<String, String> _settings = new CaseInsensitiveHashMap<>();
+    protected ContainerUser _containerUser;
+
+    private enum Settings
+    {
+        sourceRemoteSource(false),
+        sourceContainerPath(false),
+        sourceSchema(true),
+        sourceQuery(true),
+        sourceColumn(true),
+        sourceAdditionalFilters(false),
+
+        destSchema(true),
+        destQuery(true),
+        destColumn(true),
+        destAdditionalFilters(false);
+
+        private final boolean _isRequired;
+
+        Settings(boolean isRequired)
+        {
+            _isRequired = isRequired;
+        }
+
+        public boolean isRequired()
+        {
+            return _isRequired;
+        }
+    }
+
+    @Override
+    public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException
+    {
+        verifyRows(job);
+
+        return new RecordedActionSet();
+    }
+
+    @Override
+    public void setContainerUser(ContainerUser containerUser)
+    {
+        _containerUser = containerUser;
+    }
+
+    @Override
+    public List<ValidationError> preFlightCheck(Container c)
+    {
+        List<ValidationError> errors = new ArrayList<>();
+        for (String setting : getRequiredSettings())
+        {
+            if (_settings.get(setting) == null || StringUtils.isEmpty(_settings.get(setting)))
+            {
+                errors.add(new SimpleValidationError("Missing required setting: " + setting));
+            }
+        }
+
+        return errors;
+    }
+
+    @Override
+    public List<String> getRequiredSettings()
+    {
+        return Arrays.stream(Settings.values()).filter(Settings::isRequired).map(Settings::name).toList();
+    }
+
+    @Override
+    public void setSettings(Map<String, String> settings)
+    {
+        _settings.putAll(settings);
+    }
+
+    private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Container c, Logger log) throws IllegalStateException
+    {
+        DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, c, log);
+        if (rc == null)
+        {
+            throw new IllegalStateException("Unable to find remote connection: " + name);
+        }
+
+        return rc;
+    }
+
+    private long getDestinationTableCount()
+    {
+        UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), _settings.get(Settings.destSchema.name()));
+        if (us == null)
+        {
+            throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.destSchema.name()));
+        }
+
+        TableInfo ti = us.getTable(_settings.get(Settings.destQuery.name()));
+        if (ti == null)
+        {
+            throw new IllegalStateException("Unknown table: " + _settings.get(Settings.destQuery.name()));
+        }
+
+        final SimpleFilter filter = new SimpleFilter();
+        if (_settings.get(Settings.destAdditionalFilters.name()) != null)
+        {
+            if (_settings.get(Settings.destAdditionalFilters.name()) != null)
+            {
+                List<CompareType.AbstractCompareClause> additionalFilters = parseAdditionalFilters(_settings.get(Settings.destAdditionalFilters.name()));
+                additionalFilters.forEach(filter::addCondition);
+            }
+        }
+
+        TableSelector ts = new TableSelector(ti, PageFlowUtil.set(_settings.get(Settings.destColumn.name())), filter, null);
+
+        return ts.getRowCount();
+    }
+
+    private long getSourceTableCount(PipelineJob job) throws PipelineJobException
+    {
+        if (_settings.get(Settings.sourceRemoteSource.name()) != null)
+        {
+            DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.sourceRemoteSource.name()), _containerUser.getContainer(), job.getLogger());
+            SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.sourceSchema.name()), _settings.get(Settings.sourceQuery.name()));
+            sr.setColumns(Collections.singletonList(_settings.get(Settings.sourceColumn.name())));
+
+            if (_settings.get(Settings.sourceAdditionalFilters.name()) != null)
+            {
+                List<CompareType.AbstractCompareClause> additionalFilters = parseAdditionalFilters(_settings.get(Settings.sourceAdditionalFilters.name()));
+                for (CompareType.AbstractCompareClause f : additionalFilters)
+                {
+                    Object value;
+                    if (f.getParamVals() == null)
+                    {
+                        value = null;
+                    }
+                    else if (f.getParamVals().length == 1)
+                    {
+                        value = f.getParamVals()[0];
+                    }
+                    else
+                    {
+                        value = StringUtils.join(f.getParamVals(), ";");
+                    }
+
+                    Filter.Operator o = Filter.Operator.getOperatorFromUrlKey(f.getCompareType().getPreferredUrlKey());
+                    if (o == null)
+                    {
+                        throw new IllegalStateException("Unknown operator: " + f.getCompareType().getPreferredUrlKey() + ", raw filter: " + f.getCompareType().name());
+                    }
+
+                    sr.addFilter(new Filter(f.getFieldKey().toString(), value, o));
+                }
+            }
+
+            try
+            {
+                SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer);
+
+                return srr.getRowCount().longValue();
+            }
+            catch (IOException | CommandException e)
+            {
+                throw new PipelineJobException(e);
+            }
+        }
+        else
+        {
+            UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), _settings.get(Settings.sourceSchema.name()));
+            if (us == null)
+            {
+                throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.sourceSchema.name()));
+            }
+
+            TableInfo ti = us.getTable(_settings.get(Settings.sourceQuery.name()));
+            if (ti == null)
+            {
+                throw new IllegalStateException("Unknown table: " + _settings.get(Settings.sourceQuery.name()));
+            }
+
+            final SimpleFilter filter = new SimpleFilter();
+            if (_settings.get(Settings.sourceAdditionalFilters.name()) != null)
+            {
+                List<CompareType.AbstractCompareClause> additionalFilters = parseAdditionalFilters(_settings.get(Settings.sourceAdditionalFilters.name()));
+                additionalFilters.forEach(filter::addCondition);
+            }
+
+            TableSelector ts = new TableSelector(ti, PageFlowUtil.set(_settings.get(Settings.sourceColumn.name())), filter, null);
+
+            return ts.getRowCount();
+        }
+    }
+
+    private List<CompareType.AbstractCompareClause> parseAdditionalFilters(String rawVal)
+    {
+        rawVal = StringUtils.trimToNull(rawVal);
+        if (rawVal == null)
+        {
+            return Collections.emptyList();
+        }
+
+        SimpleFilter filter = new SimpleFilter();
+        String[] filters = rawVal.split(";");
+        for (String queryParam : filters)
+        {
+            filter.addUrlFilters(new ActionURL().setRawQuery(queryParam), null);
+        }
+
+        return filter.getClauses().stream().map(fc -> {
+            if (fc instanceof CompareType.AbstractCompareClause cc)
+            {
+                return cc;
+            }
+
+            throw new IllegalStateException("Expected all filters to be instance CompareType.AbstractCompareClause, found: " + fc.getClass());
+        }).toList();
+    }
+
+    private void verifyRows(PipelineJob job) throws PipelineJobException
+    {
+        job.getLogger().info("Verifying row count for: " + _settings.get(Settings.destSchema.name()) + "." + _settings.get(Settings.destQuery.name()));
+
+        long source = getSourceTableCount(job);
+        job.getLogger().info("Source (" + _settings.get(Settings.sourceSchema.name()) + "." + _settings.get(Settings.sourceQuery.name()) + ") row count: " + source);
+
+        long  dest = getDestinationTableCount();
+        job.getLogger().info("Destination (" + _settings.get(Settings.destSchema.name()) + "." + _settings.get(Settings.destQuery.name()) + ") row count: " + source);
+
+        if (source != dest)
+        {
+            job.getLogger().error("Row counts do not match!");
+        }
+    }
+}

From a93767f9c8332deea5b7be0bd3a9e077b19be6b8 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sun, 10 Aug 2025 05:49:09 -0700
Subject: [PATCH 17/24] Use maxRows in SelectRowsCommand

---
 primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
index 331c37dea..f4660bb0e 100644
--- a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
+++ b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
@@ -153,6 +153,7 @@ private long getSourceTableCount(PipelineJob job) throws PipelineJobException
             DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.sourceRemoteSource.name()), _containerUser.getContainer(), job.getLogger());
             SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.sourceSchema.name()), _settings.get(Settings.sourceQuery.name()));
             sr.setColumns(Collections.singletonList(_settings.get(Settings.sourceColumn.name())));
+            sr.setMaxRows(1); // Ensures we get rowCount
 
             if (_settings.get(Settings.sourceAdditionalFilters.name()) != null)
             {

From 5c193294b67582691a94aa9072783bc9a3518065 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 11 Aug 2025 10:45:21 -0700
Subject: [PATCH 18/24] Update ETL batchSize

---
 PMR/resources/etls/pmr-datasets.xml                      | 4 ++--
 primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml
index ebb5aa1de..6b915703c 100644
--- a/PMR/resources/etls/pmr-datasets.xml
+++ b/PMR/resources/etls/pmr-datasets.xml
@@ -148,7 +148,7 @@
                     <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
                 </sourceFilters>
             </source>
-            <destination schemaName="study" queryName="parentage" targetOption="merge" bulkLoad="true" batchSize="2500">
+            <destination schemaName="study" queryName="parentage" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -194,7 +194,7 @@
                     <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
                 </sourceFilters>
             </source>
-            <destination schemaName="study" queryName="birth" targetOption="merge" bulkLoad="true" batchSize="2500">
+            <destination schemaName="study" queryName="birth" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
diff --git a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
index f4660bb0e..95d0c2f2c 100644
--- a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
+++ b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
@@ -255,11 +255,11 @@ private void verifyRows(PipelineJob job) throws PipelineJobException
         job.getLogger().info("Source (" + _settings.get(Settings.sourceSchema.name()) + "." + _settings.get(Settings.sourceQuery.name()) + ") row count: " + source);
 
         long  dest = getDestinationTableCount();
-        job.getLogger().info("Destination (" + _settings.get(Settings.destSchema.name()) + "." + _settings.get(Settings.destQuery.name()) + ") row count: " + source);
+        job.getLogger().info("Destination (" + _settings.get(Settings.destSchema.name()) + "." + _settings.get(Settings.destQuery.name()) + ") row count: " + dest);
 
         if (source != dest)
         {
-            job.getLogger().error("Row counts do not match!");
+            job.getLogger().error("Row counts do not match (source: {}, dest: {})!", source, dest);
         }
     }
 }

From 61ee67821d69df85a70c4beac67903ce08817732 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 12 Aug 2025 15:42:56 -0700
Subject: [PATCH 19/24] Simplify mGAP ETL

---
 mGAP/resources/etls/clinicalData.xml | 140 +++++----------------------
 1 file changed, 26 insertions(+), 114 deletions(-)

diff --git a/mGAP/resources/etls/clinicalData.xml b/mGAP/resources/etls/clinicalData.xml
index 1cdc49ba8..e4801d7ef 100644
--- a/mGAP/resources/etls/clinicalData.xml
+++ b/mGAP/resources/etls/clinicalData.xml
@@ -3,23 +3,6 @@
     <name>ClinicalData</name>
     <description>EHR Clinical Data</description>
     <transforms>
-        <transform id="dataset1" type="RemoteQueryTransformStep">
-            <source schemaName="lists" queryName="chemistryResults" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>ageAtTime</column>
-                    <column>testId</column>
-                    <column>result</column>
-                    <column>units</column>
-                    <column>runid</column>
-                    <column>objectid</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="chemistryResults" targetOption="merge" bulkLoad="true" batchSize="2500">
-
-            </destination>
-        </transform>
         <transform id="dataset2" type="RemoteQueryTransformStep">
             <source schemaName="lists" queryName="demographics" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">
                 <sourceColumns>
@@ -33,111 +16,40 @@
                     <column>avgKinship</column>
                     <column>objectid</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="demographics" targetOption="merge" bulkLoad="true" batchSize="2500">
 
             </destination>
         </transform>
-        <transform id="dataset3" type="RemoteQueryTransformStep">
-            <source schemaName="lists" queryName="microbiology" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>ageAtTime</column>
-                    <column>tissue</column>
-                    <column>organism</column>
-                    <column>quantity</column>
-                    <column>runid</column>
-                    <column>objectid</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="microbiology" targetOption="merge" bulkLoad="true" batchSize="2500">
+        <transform id="rc_2" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="lists"/>
+                    <setting name="sourceQuery" value="demographics"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
 
-            </destination>
-        </transform>
-        <transform id="dataset4" type="RemoteQueryTransformStep">
-            <source schemaName="lists" queryName="weight" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>ageAtTime</column>
-                    <column>weight</column>
-                    <column>objectid</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="weight" targetOption="merge" bulkLoad="true" batchSize="2500">
-
-            </destination>
-        </transform>
-        <transform id="dataset5" type="RemoteQueryTransformStep">
-            <source schemaName="lists" queryName="clinpathRuns" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>ageAtTime</column>
-                    <column>type</column>
-                    <column>tissue</column>
-                    <column>units</column>
-                    <column>servicerequested</column>
-                    <column>collectionMethod</column>
-                    <column>method</column>
-                    <column>objectid</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="clinpathRuns" bulkLoad="true" targetOption="merge">
-
-            </destination>
-        </transform>
-        <transform id="dataset6" type="RemoteQueryTransformStep">
-            <source schemaName="lists" queryName="hematologyResults" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>ageAtTime</column>
-                    <column>testid</column>
-                    <column>result</column>
-                    <column>units</column>
-                    <column>runid</column>
-                    <column>objectid</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="hematologyResults" targetOption="merge" bulkLoad="true" batchSize="2500">
-
-            </destination>
-        </transform>
-        <transform id="dataset7" type="RemoteQueryTransformStep">
-            <source schemaName="lists" queryName="pathologyDiagnoses" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>ageAtTime</column>
-                    <column>sort_order</column>
-                    <column>codes</column>
-                    <column>objectid</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="pathologyDiagnoses" targetOption="merge" bulkLoad="true" batchSize="2500">
-
-            </destination>
-        </transform>
-        <transform id="dataset8" type="RemoteQueryTransformStep">
-            <source schemaName="lists" queryName="histology" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>ageAtTime</column>
-                    <column>sort_order</column>
-                    <column>tissue</column>
-                    <column>codes</column>
-                    <column>objectid</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="histology" targetOption="merge" bulkLoad="true" batchSize="2500">
-
-            </destination>
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="demographics"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
         </transform>
     </transforms>
-    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" pkColumnName="objectid"/>
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" pkColumnName="objectid">
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
     <schedule>
         <cron expression="0 30 1 * * ?"/>
     </schedule>

From 48bd615b86e6279dda1e1030e6ef082b52be3fcf Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 12 Aug 2025 15:56:12 -0700
Subject: [PATCH 20/24] Many improvements to EHR-related ETLs

---
 PMR/resources/etls/pmr-datasets.xml           | 226 -------
 PMR/resources/etls/pmrClinicalData.xml        |  79 ++-
 PMR/resources/etls/prime-birth.xml            |  66 ++
 PMR/resources/etls/prime-blooddraws.xml       |  52 ++
 PMR/resources/etls/prime-chemistryResults.xml |  55 ++
 PMR/resources/etls/prime-clinpathRuns.xml     |  56 ++
 ...emographics.xml => prime-demographics.xml} |  40 +-
 .../etls/prime-hematologyResults.xml          |  55 ++
 PMR/resources/etls/prime-histology.xml        |  54 ++
 PMR/resources/etls/prime-microbiology.xml     |  55 ++
 PMR/resources/etls/prime-parentage.xml        |  58 ++
 .../etls/prime-pathologyDiagnoses.xml         |  53 ++
 PMR/resources/etls/prime-weight.xml           |  55 ++
 .../study/datasets/datasets_manifest.xml      |  25 +-
 .../study/datasets/datasets_metadata.xml      | 607 ++++++++++++++----
 PMR/src/org/labkey/pmr/PMRController.java     |   4 +-
 mGAP/resources/etls/project_usage.xml         |  27 -
 primeseq/resources/etls/prime-blooddraws.xml  |  30 -
 .../resources/etls/prime-demographics.xml     |  34 -
 primeseq/resources/etls/prime-parentage.xml   |  32 -
 primeseq/resources/etls/prime-weight.xml      |  29 -
 primeseq/resources/etls/prime.xml             |  43 --
 22 files changed, 1182 insertions(+), 553 deletions(-)
 delete mode 100644 PMR/resources/etls/pmr-datasets.xml
 create mode 100644 PMR/resources/etls/prime-birth.xml
 create mode 100644 PMR/resources/etls/prime-blooddraws.xml
 create mode 100644 PMR/resources/etls/prime-chemistryResults.xml
 create mode 100644 PMR/resources/etls/prime-clinpathRuns.xml
 rename PMR/resources/etls/{pmr-demographics.xml => prime-demographics.xml} (52%)
 create mode 100644 PMR/resources/etls/prime-hematologyResults.xml
 create mode 100644 PMR/resources/etls/prime-histology.xml
 create mode 100644 PMR/resources/etls/prime-microbiology.xml
 create mode 100644 PMR/resources/etls/prime-parentage.xml
 create mode 100644 PMR/resources/etls/prime-pathologyDiagnoses.xml
 create mode 100644 PMR/resources/etls/prime-weight.xml
 delete mode 100644 mGAP/resources/etls/project_usage.xml
 delete mode 100644 primeseq/resources/etls/prime-blooddraws.xml
 delete mode 100644 primeseq/resources/etls/prime-demographics.xml
 delete mode 100644 primeseq/resources/etls/prime-parentage.xml
 delete mode 100644 primeseq/resources/etls/prime-weight.xml
 delete mode 100644 primeseq/resources/etls/prime.xml

diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml
deleted file mode 100644
index 6b915703c..000000000
--- a/PMR/resources/etls/pmr-datasets.xml
+++ /dev/null
@@ -1,226 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<etl xmlns="http://labkey.org/etl/xml" standalone="false">
-    <name>PRIMe_Datasets</name>
-    <description>PRIMe Dataset Data</description>
-    <transforms>
-<!--        <transform id="dataset1" type="RemoteQueryTransformStep">-->
-<!--            <source schemaName="study" queryName="chemistryResults" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">-->
-<!--                <sourceColumns>-->
-<!--                    <column>Id</column>-->
-<!--                    <column>date</column>-->
-<!--                    <column>ageAtTime</column>-->
-<!--                    <column>testId</column>-->
-<!--                    <column>result</column>-->
-<!--                    <column>units</column>-->
-<!--                    <column>runid</column>-->
-<!--                    <column>objectid</column>-->
-<!--                </sourceColumns>-->
-<!--            </source>-->
-<!--            <destination schemaName="study" queryName="chemistryResults" targetOption="merge" bulkLoad="true" batchSize="2500">-->
-<!--                <alternateKeys>-->
-<!--                    <column name="objectid"/>-->
-<!--                </alternateKeys>-->
-<!--            </destination>-->
-<!--        </transform>-->
-<!--        <transform id="dataset3" type="RemoteQueryTransformStep">-->
-<!--            <source schemaName="study" queryName="microbiology" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">-->
-<!--                <sourceColumns>-->
-<!--                    <column>Id</column>-->
-<!--                    <column>date</column>-->
-<!--                    <column>ageAtTime</column>-->
-<!--                    <column>tissue</column>-->
-<!--                    <column>organism</column>-->
-<!--                    <column>quantity</column>-->
-<!--                    <column>runid</column>-->
-<!--                    <column>objectid</column>-->
-<!--                </sourceColumns>-->
-<!--            </source>-->
-<!--            <destination schemaName="study" queryName="microbiology" targetOption="merge" bulkLoad="true" batchSize="2500">-->
-<!--                <alternateKeys>-->
-<!--                    <column name="objectid"/>-->
-<!--                </alternateKeys>-->
-<!--            </destination>-->
-<!--        </transform>-->
-<!--        <transform id="dataset4" type="RemoteQueryTransformStep">-->
-<!--            <source schemaName="study" queryName="weight" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">-->
-<!--                <sourceColumns>-->
-<!--                    <column>Id</column>-->
-<!--                    <column>date</column>-->
-<!--                    <column>ageAtTime</column>-->
-<!--                    <column>weight</column>-->
-<!--                    <column>objectid</column>-->
-<!--                </sourceColumns>-->
-<!--            </source>-->
-<!--            <destination schemaName="study" queryName="weight" targetOption="merge" bulkLoad="true" batchSize="2500">-->
-<!--                <alternateKeys>-->
-<!--                    <column name="objectid"/>-->
-<!--                </alternateKeys>-->
-<!--            </destination>-->
-<!--        </transform>-->
-<!--        <transform id="dataset5" type="RemoteQueryTransformStep">-->
-<!--            <source schemaName="study" queryName="clinpathRuns" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">-->
-<!--                <sourceColumns>-->
-<!--                    <column>Id</column>-->
-<!--                    <column>date</column>-->
-<!--                    <column>ageAtTime</column>-->
-<!--                    <column>type</column>-->
-<!--                    <column>tissue</column>-->
-<!--                    <column>units</column>-->
-<!--                    <column>servicerequested</column>-->
-<!--                    <column>collectionMethod</column>-->
-<!--                    <column>method</column>-->
-<!--                    <column>objectid</column>-->
-<!--                </sourceColumns>-->
-<!--            </source>-->
-<!--            <destination schemaName="study" queryName="clinpathRuns" bulkLoad="true" targetOption="merge">-->
-<!--                <alternateKeys>-->
-<!--                    <column name="objectid"/>-->
-<!--                </alternateKeys>-->
-<!--            </destination>-->
-<!--        </transform>-->
-<!--        <transform id="dataset6" type="RemoteQueryTransformStep">-->
-<!--            <source schemaName="study" queryName="hematologyResults" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">-->
-<!--                <sourceColumns>-->
-<!--                    <column>Id</column>-->
-<!--                    <column>date</column>-->
-<!--                    <column>ageAtTime</column>-->
-<!--                    <column>testid</column>-->
-<!--                    <column>result</column>-->
-<!--                    <column>units</column>-->
-<!--                    <column>runid</column>-->
-<!--                    <column>objectid</column>-->
-<!--                </sourceColumns>-->
-<!--            </source>-->
-<!--            <destination schemaName="study" queryName="hematologyResults" targetOption="merge" bulkLoad="true" batchSize="2500">-->
-<!--                <alternateKeys>-->
-<!--                    <column name="objectid"/>-->
-<!--                </alternateKeys>-->
-<!--            </destination>-->
-<!--        </transform>-->
-<!--        <transform id="dataset7" type="RemoteQueryTransformStep">-->
-<!--            <source schemaName="study" queryName="pathologyDiagnoses" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">-->
-<!--                <sourceColumns>-->
-<!--                    <column>Id</column>-->
-<!--                    <column>date</column>-->
-<!--                    <column>ageAtTime</column>-->
-<!--                    <column>sort_order</column>-->
-<!--                    <column>codes</column>-->
-<!--                    <column>objectid</column>-->
-<!--                </sourceColumns>-->
-<!--            </source>-->
-<!--            <destination schemaName="study" queryName="pathologyDiagnoses" targetOption="merge" bulkLoad="true" batchSize="2500">-->
-<!--                <alternateKeys>-->
-<!--                    <column name="objectid"/>-->
-<!--                </alternateKeys>-->
-<!--            </destination>-->
-<!--        </transform>-->
-<!--        <transform id="dataset8" type="RemoteQueryTransformStep">-->
-<!--            <source schemaName="study" queryName="histology" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">-->
-<!--                <sourceColumns>-->
-<!--                    <column>Id</column>-->
-<!--                    <column>date</column>-->
-<!--                    <column>ageAtTime</column>-->
-<!--                    <column>sort_order</column>-->
-<!--                    <column>tissue</column>-->
-<!--                    <column>codes</column>-->
-<!--                    <column>objectid</column>-->
-<!--                </sourceColumns>-->
-<!--            </source>-->
-<!--            <destination schemaName="study" queryName="histology" targetOption="merge" bulkLoad="true" batchSize="2500">-->
-<!--                <alternateKeys>-->
-<!--                    <column name="objectid"/>-->
-<!--                </alternateKeys>-->
-<!--            </destination>-->
-<!--        </transform>-->
-        <transform id="dataset9" type="RemoteQueryTransformStep">
-            <source schemaName="study" queryName="parentage" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>parent</column>
-                    <column>relationship</column>
-                    <column>method</column>
-                    <column>objectid</column>
-                    <column>created</column>
-                    <column>modified</column>
-                </sourceColumns>
-                <sourceFilters>
-                    <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
-                </sourceFilters>
-            </source>
-            <destination schemaName="study" queryName="parentage" targetOption="merge" bulkLoad="true" batchSize="1000">
-                <alternateKeys>
-                    <column name="objectid"/>
-                </alternateKeys>
-            </destination>
-        </transform>
-        <transform id="dataset9_rc" type="TaskrefTransformStep">
-            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
-                <settings>
-                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
-                    <setting name="sourceSchema" value="study"/>
-                    <setting name="sourceQuery" value="parentage"/>
-                    <setting name="sourceColumn" value="objectId"/>
-
-                    <setting name="destSchema" value="study"/>
-                    <setting name="destQuery" value="parentage"/>
-                    <setting name="destColumn" value="objectId"/>
-                </settings>
-            </taskref>
-        </transform>
-        <transform id="dataset10" type="RemoteQueryTransformStep">
-            <source schemaName="study" queryName="birth" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>date_type</column>
-                    <column>birth_condition</column>
-                    <column>dam</column>
-                    <column>gender</column>
-                    <column>weight</column>
-                    <column>wdate</column>
-                    <column>type</column>
-                    <column>species</column>
-                    <column>geographic_origin</column>
-                    <column>sire</column>
-                    <column>estimated</column>
-                    <column>conception</column>
-                    <column>conceptualDay</column>
-                    <column>objectid</column>
-                    <column>created</column>
-                    <column>modified</column>
-                </sourceColumns>
-                <sourceFilters>
-                    <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
-                </sourceFilters>
-            </source>
-            <destination schemaName="study" queryName="birth" targetOption="merge" bulkLoad="true" batchSize="1000">
-                <alternateKeys>
-                    <column name="objectid"/>
-                </alternateKeys>
-            </destination>
-        </transform>
-        <transform id="dataset10_rc" type="TaskrefTransformStep">
-            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
-                <settings>
-                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
-                    <setting name="sourceSchema" value="study"/>
-                    <setting name="sourceQuery" value="birth"/>
-                    <setting name="sourceColumn" value="objectId"/>
-
-                    <setting name="destSchema" value="study"/>
-                    <setting name="destQuery" value="birth"/>
-                    <setting name="destColumn" value="objectId"/>
-                </settings>
-            </taskref>
-        </transform>
-    </transforms>
-
-    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="Created" >
-        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid" timestampColumnName="Created"/>
-    </incrementalFilter>
-    <schedule>
-        <!--11PM-->
-        <cron expression="0 0 23 * * ?"/>
-    </schedule>
-</etl>
diff --git a/PMR/resources/etls/pmrClinicalData.xml b/PMR/resources/etls/pmrClinicalData.xml
index 591827c32..735c44f24 100644
--- a/PMR/resources/etls/pmrClinicalData.xml
+++ b/PMR/resources/etls/pmrClinicalData.xml
@@ -6,18 +6,91 @@
         <transform id="demographics" type="TaskrefTransformStep">
             <taskref ref="org.labkey.di.steps.QueueJobTask">
                 <settings>
-                    <setting name="transformId" value="{PMR}/pmr-demographics"/>
+                    <setting name="transformId" value="{PMR}/prime-demographics"/>
                 </settings>
             </taskref>
         </transform>
 
-        <transform id="datasets" type="TaskrefTransformStep">
+        <transform id="birth" type="TaskrefTransformStep">
             <taskref ref="org.labkey.di.steps.QueueJobTask">
                 <settings>
-                    <setting name="transformId" value="{PMR}/pmr-datasets"/>
+                    <setting name="transformId" value="{PMR}/prime-birth"/>
                 </settings>
             </taskref>
         </transform>
+
+        <transform id="blooddraws" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-blooddraws"/>
+                </settings>
+            </taskref>
+        </transform>
+
+        <transform id="chemistryResults" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-chemistryResults"/>
+                </settings>
+            </taskref>
+        </transform>
+
+        <transform id="clinpathRuns" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-clinpathRuns"/>
+                </settings>
+            </taskref>
+        </transform>
+
+        <transform id="hematologyResults" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-hematologyResults"/>
+                </settings>
+            </taskref>
+        </transform>
+
+        <transform id="histology" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-histology"/>
+                </settings>
+            </taskref>
+        </transform>
+
+        <transform id="microbiology" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-microbiology"/>
+                </settings>
+            </taskref>
+        </transform>
+
+        <transform id="parentage" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-parentage"/>
+                </settings>
+            </taskref>
+        </transform>
+
+        <transform id="pathologyDiagnoses" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-pathologyDiagnoses"/>
+                </settings>
+            </taskref>
+        </transform>
+
+        <transform id="weight" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.di.steps.QueueJobTask">
+                <settings>
+                    <setting name="transformId" value="{PMR}/prime-weight"/>
+                </settings>
+            </taskref>
+        </transform>
+
     </transforms>
     <schedule>
         <cron expression="0 30 1 * * ?"/>
diff --git a/PMR/resources/etls/prime-birth.xml b/PMR/resources/etls/prime-birth.xml
new file mode 100644
index 000000000..4556dba00
--- /dev/null
+++ b/PMR/resources/etls/prime-birth.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_Birth_Data</name>
+    <description>PRIMe Birth Data</description>
+    <transforms>
+        <transform id="select" type="RemoteQueryTransformStep">
+            <source schemaName="study" queryName="birth" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>date_type</column>
+                    <column>birth_condition</column>
+                    <column>dam</column>
+                    <column>gender</column>
+                    <column>weight</column>
+                    <column>wdate</column>
+                    <column>type</column>
+                    <column>species</column>
+                    <column>geographic_origin</column>
+                    <column>sire</column>
+                    <column>estimated</column>
+                    <column>conception</column>
+                    <column>conceptualDay</column>
+                    <column>objectid</column>
+                    <column>created</column>
+                    <column>QCState/Label</column>
+                </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
+                </sourceFilters>
+            </source>
+            <destination schemaName="study" queryName="birth" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="birth"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="birth"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="birth" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/prime-blooddraws.xml b/PMR/resources/etls/prime-blooddraws.xml
new file mode 100644
index 000000000..13abeaa15
--- /dev/null
+++ b/PMR/resources/etls/prime-blooddraws.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_Blood_Data</name>
+    <description>PRIMe Blood Data</description>
+    <transforms>
+        <transform type="RemoteQueryTransformStep" id="blood">
+            <description>Copy to target</description>
+            <source remoteSource="EHR_ClinicalSource" schemaName="study" queryName="blood" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>reason</column>
+                    <column>quantity</column>
+                    <column>objectid</column>
+                    <column>QCState/Label</column>
+                    <column>created</column>
+                    <column>modified</column>
+                </sourceColumns>
+            </source>
+            <destination schemaName="study" queryName="blood" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="blood"/>
+                    <setting name="sourceColumn" value="objectId"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="blood"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="blood" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
new file mode 100644
index 000000000..0c0cd1c92
--- /dev/null
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_ChemistryResults_Data</name>
+    <description>PRIMe Chemistry Results Data</description>
+    <transforms>
+        <transform id="select" type="RemoteQueryTransformStep">
+            <source schemaName="study" queryName="chemistryResults" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>ageAtTime</column>
+                    <column>testId</column>
+                    <column>result</column>
+                    <column>units</column>
+                    <column>runid</column>
+                    <column>objectid</column>
+                    <column>created</column>
+                    <column>modified</column>
+                    <column>QCState/Label</column>
+                </sourceColumns>
+            </source>
+            <destination schemaName="study" queryName="chemistryResults" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="chemistryResults"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="chemistryResults"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="chemistryResults" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/prime-clinpathRuns.xml b/PMR/resources/etls/prime-clinpathRuns.xml
new file mode 100644
index 000000000..f6afe0b27
--- /dev/null
+++ b/PMR/resources/etls/prime-clinpathRuns.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_ClinpathRuns_Data</name>
+    <description>PRIMe ClinpathRuns Data</description>
+    <transforms>
+        <transform id="select" type="RemoteQueryTransformStep">
+            <source schemaName="study" queryName="clinpathRuns" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>type</column>
+                    <column>tissue</column>
+                    <column>units</column>
+                    <column>servicerequested</column>
+                    <column>collectionMethod</column>
+                    <column>method</column>
+                    <column>objectid</column>
+                    <column>created</column>
+                    <column>modified</column>
+                    <column>QCState/Label</column>
+                </sourceColumns>
+            </source>
+            <destination schemaName="study" queryName="clinpathRuns" bulkLoad="true" targetOption="merge" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="clinpathRuns"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="clinpathRuns"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="clinpathRuns" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/pmr-demographics.xml b/PMR/resources/etls/prime-demographics.xml
similarity index 52%
rename from PMR/resources/etls/pmr-demographics.xml
rename to PMR/resources/etls/prime-demographics.xml
index 883f477ce..87fb1a758 100644
--- a/PMR/resources/etls/pmr-demographics.xml
+++ b/PMR/resources/etls/prime-demographics.xml
@@ -1,42 +1,44 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <etl xmlns="http://labkey.org/etl/xml" standalone="false">
-    <name>PRIMe_Demographics</name>
+    <name>PRIME_Demographics_Data</name>
     <description>PRIMe Demographics Data</description>
     <transforms>
-        <transform id="dataset1" type="RemoteQueryTransformStep">
-            <source schemaName="study" queryName="demographics" remoteSource="EHR_ClinicalSource" timestampColumnName="modified">
+        <transform type="RemoteQueryTransformStep" id="select">
+            <description>Copy to target</description>
+            <source remoteSource="EHR_ClinicalSource" schemaName="study" queryName="Demographics" timestampColumnName="modified" sourceTimeout="0">
                 <sourceColumns>
                     <column>Id</column>
                     <column>date</column>
                     <column>gender</column>
-                    <column>species</column>
                     <column>geographic_origin</column>
                     <column>birth</column>
                     <column>death</column>
+                    <column>species</column>
                     <column>status</column>
                     <column>calculated_status</column>
-                    <column>QCState/Label</column>
                     <column>objectid</column>
+                    <column>QCState/Label</column>
                     <column>created</column>
                     <column>modified</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
-            <destination schemaName="study" queryName="demographics" targetOption="truncate" bulkLoad="true" batchSize="2500">
-                <columnTransforms>
-                    <column source="QCState/Label" target="QCStateLabel"/>
-                </columnTransforms>
+            <destination schemaName="study" queryName="Demographics" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
-                    <column name="Id"/>
+                    <column name="objectid"/>
                 </alternateKeys>
             </destination>
         </transform>
-        <transform id="dataset9_rc" type="TaskrefTransformStep">
+        <transform id="rc" type="TaskrefTransformStep">
             <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
                 <settings>
                     <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
                     <setting name="sourceSchema" value="study"/>
                     <setting name="sourceQuery" value="demographics"/>
                     <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
 
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="demographics"/>
@@ -45,9 +47,15 @@
             </taskref>
         </transform>
     </transforms>
-
-    <schedule>
-        <!--11PM-->
-        <cron expression="0 0 23 * * ?"/>
-    </schedule>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="demographics" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
 </etl>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
new file mode 100644
index 000000000..9df7326e3
--- /dev/null
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_HematologyResults_Data</name>
+    <description>PRIMe Hematology Results Data</description>
+    <transforms>
+        <transform id="select" type="RemoteQueryTransformStep">
+            <source schemaName="study" queryName="hematologyResults" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>ageAtTime</column>
+                    <column>testId</column>
+                    <column>result</column>
+                    <column>units</column>
+                    <column>runid</column>
+                    <column>objectid</column>
+                    <column>created</column>
+                    <column>modified</column>
+                    <column>QCState/Label</column>
+                </sourceColumns>
+            </source>
+            <destination schemaName="study" queryName="hematologyResults" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="hematologyResults"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="hematologyResults"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="hematologyResults" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/prime-histology.xml b/PMR/resources/etls/prime-histology.xml
new file mode 100644
index 000000000..27a21350b
--- /dev/null
+++ b/PMR/resources/etls/prime-histology.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_Histology_Data</name>
+    <description>PRIMe Histology Data</description>
+    <transforms>
+        <transform id="select" type="RemoteQueryTransformStep">
+            <source schemaName="study" queryName="histology" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>ageAtTime</column>
+                    <column>sort_order</column>
+                    <column>tissue</column>
+                    <column>codes</column>
+                    <column>objectid</column>
+                    <column>created</column>
+                    <column>modified</column>
+                    <column>QCState/Label</column>
+                </sourceColumns>
+            </source>
+            <destination schemaName="study" queryName="histology" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="histology"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="histology"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="histology" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/prime-microbiology.xml b/PMR/resources/etls/prime-microbiology.xml
new file mode 100644
index 000000000..194c4d83e
--- /dev/null
+++ b/PMR/resources/etls/prime-microbiology.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_Microbiology_Data</name>
+    <description>PRIMe Microbiology Data</description>
+    <transforms>
+        <transform id="select" type="RemoteQueryTransformStep">
+            <source schemaName="study" queryName="microbiology" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>ageAtTime</column>
+                    <column>tissue</column>
+                    <column>organism</column>
+                    <column>quantity</column>
+                    <column>runid</column>
+                    <column>objectid</column>
+                    <column>created</column>
+                    <column>modified</column>
+                    <column>QCState/Label</column>
+                </sourceColumns>
+            </source>
+            <destination schemaName="study" queryName="microbiology" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="microbiology"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="microbiology"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="microbiology" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/prime-parentage.xml b/PMR/resources/etls/prime-parentage.xml
new file mode 100644
index 000000000..30f46b3f7
--- /dev/null
+++ b/PMR/resources/etls/prime-parentage.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_Parentage_Data</name>
+    <description>PRIMe Parentage Data</description>
+    <transforms>
+        <transform type="RemoteQueryTransformStep" id="select">
+            <description>Copy to target</description>
+            <source remoteSource="EHR_ClinicalSource" schemaName="study" queryName="Parentage" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>enddate</column>
+                    <column>parent</column>
+                    <column>relationship</column>
+                    <column>method</column>
+                    <column>objectid</column>
+                    <column>QCState/Label</column>
+                    <column>created</column>
+                    <column>modified</column>
+                </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
+                </sourceFilters>
+            </source>
+            <destination schemaName="study" queryName="Parentage" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="parentage"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="parentage"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="parentage" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/prime-pathologyDiagnoses.xml b/PMR/resources/etls/prime-pathologyDiagnoses.xml
new file mode 100644
index 000000000..428ef6e3a
--- /dev/null
+++ b/PMR/resources/etls/prime-pathologyDiagnoses.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_PathologyDiagnoses_Data</name>
+    <description>PRIMe Pathology Diagnoses Data</description>
+    <transforms>
+        <transform id="select" type="RemoteQueryTransformStep">
+            <source schemaName="study" queryName="pathologyDiagnoses" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>ageAtTime</column>
+                    <column>sort_order</column>
+                    <column>codes</column>
+                    <column>objectid</column>
+                    <column>created</column>
+                    <column>modified</column>
+                    <column>QCState/Label</column>
+                </sourceColumns>
+            </source>
+            <destination schemaName="study" queryName="pathologyDiagnoses" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="pathologyDiagnoses"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="pathologyDiagnoses"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="pathologyDiagnoses" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/etls/prime-weight.xml b/PMR/resources/etls/prime-weight.xml
new file mode 100644
index 000000000..435679e81
--- /dev/null
+++ b/PMR/resources/etls/prime-weight.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml" standalone="true">
+    <name>PRIME_Weight_Data</name>
+    <description>PRIMe Weight Data</description>
+    <transforms>
+        <transform type="RemoteQueryTransformStep" id="select">
+            <description>Copy to target</description>
+            <source remoteSource="EHR_ClinicalSource" schemaName="study" queryName="weight" timestampColumnName="modified" sourceTimeout="0">
+                <sourceColumns>
+                    <column>Id</column>
+                    <column>date</column>
+                    <column>weight</column>
+                    <column>objectid</column>
+                    <column>QCState/Label</column>
+                    <column>created</column>
+                    <column>modified</column>
+                </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
+                </sourceFilters>
+            </source>
+            <destination schemaName="study" queryName="weight" targetOption="merge" bulkLoad="true" batchSize="1000">
+                <alternateKeys>
+                    <column name="objectid"/>
+                </alternateKeys>
+            </destination>
+        </transform>
+        <transform id="rc" type="TaskrefTransformStep">
+            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
+                <settings>
+                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
+                    <setting name="sourceSchema" value="study"/>
+                    <setting name="sourceQuery" value="weight"/>
+                    <setting name="sourceColumn" value="objectId"/>
+                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
+
+                    <setting name="destSchema" value="study"/>
+                    <setting name="destQuery" value="weight"/>
+                    <setting name="destColumn" value="objectId"/>
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
+        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
+            <sourceFilters>
+                <sourceFilter column="datasetid/Name" operator="eq" value="weight" />
+                <sourceFilter column="Comment" operator="contains" value="Delete"/>
+            </sourceFilters>
+            <sourceColumns>
+                <column>primaryKey</column>
+            </sourceColumns>
+        </deletedRowsSource>
+    </incrementalFilter>
+</etl>
diff --git a/PMR/resources/referenceStudy/study/datasets/datasets_manifest.xml b/PMR/resources/referenceStudy/study/datasets/datasets_manifest.xml
index a57ce797a..6ee27d416 100644
--- a/PMR/resources/referenceStudy/study/datasets/datasets_manifest.xml
+++ b/PMR/resources/referenceStudy/study/datasets/datasets_manifest.xml
@@ -28,10 +28,25 @@
         <dataset name="weight" id="1045" showByDefault="true" category="Clinical" type="Standard">
             <tags/>
         </dataset>
-        <dataset name="labwork" id="1073" showByDefault="false" category="ClinPath" type="Standard">
+        <dataset name="clinpathRuns" id="1011" category="Clinical" type="Standard" showByDefault="true">
             <tags/>
         </dataset>
-        <dataset name="labworkResults" id="1074" showByDefault="false" category="ClinPath" type="Standard">
+        <dataset name="blood" id="1008" category="Clinical" type="Standard">
+            <tags/>
+        </dataset>
+        <dataset name="microbiology" id="1015" category="Clinical" type="Standard" showByDefault="true">
+            <tags/>
+        </dataset>
+        <dataset name="hematologyResults" id="1004" category="Clinical" type="Standard" showByDefault="true">
+            <tags/>
+        </dataset>
+        <dataset name="chemistryResults" id="1005" category="Clinical" type="Standard"  showByDefault="true">
+            <tags/>
+        </dataset>
+        <dataset name="pathologyDiagnoses" id="1006" category="Clinical" type="Standard" showByDefault="false">
+            <tags/>
+        </dataset>
+        <dataset name="histology" id="1017" category="Clinical" type="Standard" showByDefault="false">
             <tags/>
         </dataset>
         <dataset name="transfers" id="1001" category="Colony Management" type="Standard">
@@ -67,5 +82,11 @@
         <dataset name="viralLoads" id="1032" showByDefault="true" category="Research" type="Standard">
             <tags/>
         </dataset>
+        <dataset name="parasitologyResults" id="1052" category="ClinPath" type="Standard">
+            <tags/>
+        </dataset>
+        <dataset name="urinalysisResults" id="1081" category="ClinPath" type="Standard">
+            <tags/>
+        </dataset>
     </datasets>
 </datasets>
diff --git a/PMR/resources/referenceStudy/study/datasets/datasets_metadata.xml b/PMR/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 042e53ace..f279b1638 100644
--- a/PMR/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/PMR/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -187,7 +187,7 @@
                 <datatype>integer</datatype>
                 <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
             </column>
-			<column columnName="type">
+            <column columnName="type">
                 <datatype>varchar</datatype>
             </column>
             <column columnName="objectid">
@@ -423,114 +423,6 @@
         </columns>
         <tableTitle>Weight</tableTitle>
     </table>
-    <table tableName="labwork" tableDbType="TABLE">
-        <columns>
-            <column columnName="Id">
-                <datatype>varchar</datatype>
-                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
-                <importAliases>
-                    <importAlias>ptid</importAlias>
-                </importAliases>
-            </column>
-            <column columnName="date">
-                <datatype>timestamp</datatype>
-                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
-                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
-            </column>
-            <column columnName="enddate">
-                <datatype>timestamp</datatype>
-                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
-            </column>
-            <column columnName="type">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="project">
-                <datatype>integer</datatype>
-                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
-            </column>
-            <column columnName="objectid">
-                <datatype>entityid</datatype>
-                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
-                <isKeyField>true</isKeyField>
-            </column>
-            <column columnName="sampletype">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="collectionMethod">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="method">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="sampleQuantity">
-                <datatype>double</datatype>
-            </column>
-            <column columnName="quantityUnits">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="datefinalized">
-                <datatype>timestamp</datatype>
-            </column>
-        </columns>
-        <tableTitle>Labwork</tableTitle>
-    </table>
-    <table tableName="labworkResults" tableDbType="TABLE">
-        <columns>
-            <column columnName="Id">
-                <datatype>varchar</datatype>
-                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
-                <importAliases>
-                    <importAlias>ptid</importAlias>
-                </importAliases>
-            </column>
-            <column columnName="date">
-                <datatype>timestamp</datatype>
-                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
-                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
-            </column>
-            <column columnName="testid">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="result">
-                <datatype>double</datatype>
-            </column>
-            <column columnName="resultOORIndicator">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="resultNumber">
-                <datatype>double</datatype>
-            </column>
-            <column columnName="resultInRange">
-                <datatype>double</datatype>
-            </column>
-            <column columnName="units">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="qualresult">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="objectid">
-                <datatype>entityid</datatype>
-                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
-                <isKeyField>true</isKeyField>
-            </column>
-            <column columnName="runid">
-                <datatype>varchar</datatype>
-            </column>
-            <column columnName="project">
-                <datatype>integer</datatype>
-                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
-            </column>
-            <column columnName="enddate">
-                <datatype>timestamp</datatype>
-                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
-            </column>
-            <column columnName="method">
-                <datatype>varchar</datatype>
-            </column>
-        </columns>
-        <tableTitle>Lab Results</tableTitle>
-    </table>
     <table tableName="transfers" tableDbType="TABLE">
         <columns>
             <column columnName="Id">
@@ -1044,4 +936,501 @@
         </columns>
         <tableTitle>Kinship</tableTitle>
     </table>
+    <table tableName="clinpathRuns" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
+            </column>
+            <column columnName="type">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="tissue">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="instructions">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="units">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="servicerequested">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="collectedBy">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="sampleId">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="sampletype">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="collectionMethod">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="method">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="sampleQuantity">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="quantityUnits">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="chargetype">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="datefinalized">
+                <datatype>timestamp</datatype>
+            </column>
+            <column columnName="vet">
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+        <tableTitle>Clinpath Runs</tableTitle>
+    </table>
+    <table tableName="hematologyResults" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="testid">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="result">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="resultOORIndicator">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="resultNumber">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="resultInRange">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="units">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="qualresult">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="runid">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
+            </column>
+            <column columnName="method">
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+        <tableTitle>Hematology Results</tableTitle>
+    </table>
+    <table tableName="parasitologyResults" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="organism">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="runid">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
+            </column>
+            <column columnName="result">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="units">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="sampletype">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="qualresult">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="method">
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+        <tableTitle>Parasitology Results</tableTitle>
+    </table>
+    <table tableName="urinalysisResults" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="testid">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="result">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="resultOORIndicator">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="resultNumber">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="resultInRange">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="rangeMax">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="units">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="qualresult">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
+            </column>
+            <column columnName="method">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="quantity">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="quantityOORIndicator">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="quantityNumber">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="quantityInRange">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="runid">
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+        <tableTitle>Urinalysis Results</tableTitle>
+    </table>
+    <table tableName="blood" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="tube_type">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="tube_vol">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="quantity">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="sampletype">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="chargetype">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="num_tubes">
+                <datatype>integer</datatype>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="additionalServices">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
+            </column>
+            <column columnName="daterequested">
+                <datatype>timestamp</datatype>
+                <propertyURI>urn:ehr.labkey.org/#DateRequested</propertyURI>
+            </column>
+            <column columnName="instructions">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="reason">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="datefinalized">
+                <datatype>timestamp</datatype>
+            </column>
+        </columns>
+        <tableTitle>Blood Draws</tableTitle>
+    </table>
+    <table tableName="pathologyDiagnoses" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="sort_order">
+                <datatype>integer</datatype>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="formSort">
+                <datatype>integer</datatype>
+            </column>
+        </columns>
+        <tableTitle>Pathology Diagnoses</tableTitle>
+    </table>
+    <table tableName="histology" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="sort_order">
+                <datatype>integer</datatype>
+            </column>
+            <column columnName="tissue">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
+            </column>
+            <column columnName="stain">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="qualifier">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="formSort">
+                <datatype>integer</datatype>
+            </column>
+        </columns>
+        <tableTitle>Histology</tableTitle>
+    </table>
+    <table tableName="microbiology" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="servicerequest">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="tissue">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="organism">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="quantity">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="runid">
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+        <tableTitle>Microbiology Results</tableTitle>
+    </table>
+    <table tableName="chemistryResults" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="testid">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="result">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="resultOORIndicator">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="resultNumber">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="resultInRange">
+                <datatype>double</datatype>
+            </column>
+            <column columnName="units">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="qualresult">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="objectid">
+                <datatype>entityid</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="runid">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="project">
+                <datatype>integer</datatype>
+                <propertyURI>urn:ehr.labkey.org/#Project</propertyURI>
+            </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+                <propertyURI>urn:ehr.labkey.org/#EndDate</propertyURI>
+            </column>
+            <column columnName="method">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="type">
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+        <tableTitle>Chemistry Results</tableTitle>
+    </table>
 </tables>
diff --git a/PMR/src/org/labkey/pmr/PMRController.java b/PMR/src/org/labkey/pmr/PMRController.java
index 69487b75d..8dfab686a 100644
--- a/PMR/src/org/labkey/pmr/PMRController.java
+++ b/PMR/src/org/labkey/pmr/PMRController.java
@@ -52,9 +52,9 @@ public static class ImportStudyAction extends ConfirmAction<Object>
         @Override
         public ModelAndView getConfirmView(Object o, BindException errors) throws Exception
         {
-            setTitle("Import mGAP Study");
+            setTitle("Import PMR Study");
 
-            return new HtmlView(HtmlString.unsafe("This will import the default mGAP study in this folder and set the EHRStudyContainer property to point to this container. Do you want to continue?"));
+            return new HtmlView(HtmlString.unsafe("This will import the default PMR study in this folder and set the EHRStudyContainer property to point to this container. Do you want to continue?"));
         }
 
         @Override
diff --git a/mGAP/resources/etls/project_usage.xml b/mGAP/resources/etls/project_usage.xml
deleted file mode 100644
index 67df09b66..000000000
--- a/mGAP/resources/etls/project_usage.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<etl xmlns="http://labkey.org/etl/xml">
-    <name>Project Usage</name>
-    <description>Project Usage Data</description>
-    <transforms>
-        <transform id="step1" type="RemoteQueryTransformStep">
-            <description>Copy to local table</description>
-            <source schemaName="laboratory" queryName="project_usage" remoteSource="PRIME_DNA_Bank">
-                <sourceColumns>
-                    <column>subjectId</column>
-                    <column>project</column>
-                    <column>groupname</column>
-                    <column>startdate</column>
-                    <column>enddate</column>
-                    <column>comment</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="laboratory" queryName="project_usage" targetOption="truncate" bulkLoad="true">
-
-            </destination>
-        </transform>
-    </transforms>
-    <schedule>
-        <!--11:20PM-->
-        <cron expression="0 30 23 * * ?"/>
-    </schedule>
-</etl>
diff --git a/primeseq/resources/etls/prime-blooddraws.xml b/primeseq/resources/etls/prime-blooddraws.xml
deleted file mode 100644
index 8faf28096..000000000
--- a/primeseq/resources/etls/prime-blooddraws.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<etl xmlns="http://labkey.org/etl/xml" standalone="false">
-    <name>PRIME_Blood_Data</name>
-    <description>PRIMe Blood Data</description>
-    <transforms>
-        <transform type="RemoteQueryTransformStep" id="blood">
-            <description>Copy to target</description>
-            <source remoteSource="PRIME" schemaName="study" queryName="blood" sourceTimeout="0">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>reason</column>
-                    <column>quantity</column>
-                    <column>objectid</column>
-                    <column>QCState</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="blood" targetOption="merge" bulkLoad="true">
-
-            </destination>
-        </transform>
-    </transforms>
-    <incrementalFilter timestampColumnName="modified" pkColumnName="lsid" className="ModifiedSinceFilterStrategy" >
-        <deletedRowsSource remoteSource="PRIME" schemaName="auditlog" queryName="DatasetAuditEvent" timestampColumnName="Created" sourceTimeout="0" deletedSourceKeyColumnName="lsid" targetKeyColumnName="lsid">
-            <sourceFilters>
-                <sourceFilter column="datasetid/Name" operator="eq" value="blood" />
-            </sourceFilters>
-        </deletedRowsSource>
-    </incrementalFilter>
-</etl>
diff --git a/primeseq/resources/etls/prime-demographics.xml b/primeseq/resources/etls/prime-demographics.xml
deleted file mode 100644
index 6485f5ee1..000000000
--- a/primeseq/resources/etls/prime-demographics.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<etl xmlns="http://labkey.org/etl/xml" standalone="false">
-    <name>PRIME_Demographics_Data</name>
-    <description>PRIMe Demographics Data</description>
-    <transforms>
-        <transform type="RemoteQueryTransformStep" id="demographics">
-            <description>Copy to target</description>
-            <source remoteSource="PRIME" schemaName="study" queryName="Demographics" sourceTimeout="0">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>gender</column>
-                    <column>geographic_origin</column>
-                    <column>birth</column>
-                    <column>death</column>
-                    <column>species</column>
-                    <column>status</column>
-                    <column>objectid</column>
-                    <column>QCState</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="Demographics" targetOption="merge" bulkLoad="true">
-
-            </destination>
-        </transform>
-    </transforms>
-    <incrementalFilter timestampColumnName="modified" pkColumnName="lsid" className="ModifiedSinceFilterStrategy" >
-        <deletedRowsSource remoteSource="PRIME" schemaName="auditlog" queryName="DatasetAuditEvent" timestampColumnName="Created" sourceTimeout="0" deletedSourceKeyColumnName="lsid" targetKeyColumnName="lsid">
-            <sourceFilters>
-                <sourceFilter column="datasetid/Name" operator="eq" value="blood" />
-            </sourceFilters>
-        </deletedRowsSource>
-    </incrementalFilter>
-</etl>
diff --git a/primeseq/resources/etls/prime-parentage.xml b/primeseq/resources/etls/prime-parentage.xml
deleted file mode 100644
index 107683da9..000000000
--- a/primeseq/resources/etls/prime-parentage.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<etl xmlns="http://labkey.org/etl/xml" standalone="false">
-    <name>PRIME_Parentage_Data</name>
-    <description>PRIMe Parentage Data</description>
-    <transforms>
-        <transform type="RemoteQueryTransformStep" id="demographics">
-            <description>Copy to target</description>
-            <source remoteSource="PRIME" schemaName="study" queryName="Parentage" sourceTimeout="0">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>parent</column>
-                    <column>relationship</column>
-                    <column>method</column>
-                    <column>datedisabled</column>
-                    <column>objectid</column>
-                    <column>QCState</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="Parentage" targetOption="merge" bulkLoad="true">
-
-            </destination>
-        </transform>
-    </transforms>
-    <incrementalFilter timestampColumnName="modified" pkColumnName="lsid" className="ModifiedSinceFilterStrategy" >
-        <deletedRowsSource remoteSource="PRIME" schemaName="auditlog" queryName="DatasetAuditEvent" timestampColumnName="Created" sourceTimeout="0" deletedSourceKeyColumnName="lsid" targetKeyColumnName="lsid">
-            <sourceFilters>
-                <sourceFilter column="datasetid/Name" operator="eq" value="parentage" />
-            </sourceFilters>
-        </deletedRowsSource>
-    </incrementalFilter>
-</etl>
diff --git a/primeseq/resources/etls/prime-weight.xml b/primeseq/resources/etls/prime-weight.xml
deleted file mode 100644
index dba86c54d..000000000
--- a/primeseq/resources/etls/prime-weight.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<etl xmlns="http://labkey.org/etl/xml" standalone="false">
-    <name>PRIME_Parentage_Data</name>
-    <description>PRIMe Parentage Data</description>
-    <transforms>
-        <transform type="RemoteQueryTransformStep" id="weight">
-            <description>Copy to target</description>
-            <source remoteSource="PRIME" schemaName="study" queryName="weight" sourceTimeout="0">
-                <sourceColumns>
-                    <column>Id</column>
-                    <column>date</column>
-                    <column>weight</column>
-                    <column>objectid</column>
-                    <column>QCState</column>
-                </sourceColumns>
-            </source>
-            <destination schemaName="study" queryName="weight" targetOption="merge" bulkLoad="true">
-
-            </destination>
-        </transform>
-    </transforms>
-    <incrementalFilter timestampColumnName="modified" pkColumnName="lsid" className="ModifiedSinceFilterStrategy" >
-        <deletedRowsSource remoteSource="PRIME" schemaName="auditlog" queryName="DatasetAuditEvent" timestampColumnName="Created" sourceTimeout="0" deletedSourceKeyColumnName="lsid" targetKeyColumnName="lsid">
-            <sourceFilters>
-                <sourceFilter column="datasetid/Name" operator="eq" value="weight" />
-            </sourceFilters>
-        </deletedRowsSource>
-    </incrementalFilter>
-</etl>
diff --git a/primeseq/resources/etls/prime.xml b/primeseq/resources/etls/prime.xml
deleted file mode 100644
index f84182ab3..000000000
--- a/primeseq/resources/etls/prime.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<etl xmlns="http://labkey.org/etl/xml">
-    <name>PRIME_Data</name>
-    <description>PRIMe Clinical/Demographics Data</description>
-    <transforms>
-        <transform id="demographics" type="TaskrefTransformStep">
-            <taskref ref="org.labkey.di.steps.QueueJobTask">
-                <settings>
-                    <setting name="transformId" value="{primeseq}/prime-demographics"/>
-                </settings>
-            </taskref>
-        </transform>
-
-        <transform id="parentage" type="TaskrefTransformStep">
-            <taskref ref="org.labkey.di.steps.QueueJobTask">
-                <settings>
-                    <setting name="transformId" value="{primeseq}/prime-parentage"/>
-                </settings>
-            </taskref>
-        </transform>
-
-        <transform id="weight" type="TaskrefTransformStep">
-            <taskref ref="org.labkey.di.steps.QueueJobTask">
-                <settings>
-                    <setting name="transformId" value="{primeseq}/prime-weight"/>
-                </settings>
-            </taskref>
-        </transform>
-
-        <transform id="blooddraws" type="TaskrefTransformStep">
-            <taskref ref="org.labkey.di.steps.QueueJobTask">
-                <settings>
-                    <setting name="transformId" value="{primeseq}/prime-blooddraws"/>
-                </settings>
-            </taskref>
-        </transform>
-
-    </transforms>
-
-    <schedule>
-        <cron expression="0 0 3 * * ?"/>
-    </schedule>
-</etl>

From ad75704324000abd1a1a5190442ba6a432b81c26 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 13 Aug 2025 06:55:46 -0700
Subject: [PATCH 21/24] Improve mGAP QC

---
 .../queries/mGAP/sequenceDatasetsSource.query.xml   | 10 ++++++++++
 .../queries/mGAP/sequenceDatasetsSource.sql         |  3 ++-
 mGAP/resources/schemas/mgap.xml                     |  1 +
 mGAP/resources/views/mgapDataDashboard.html         | 13 +++++++++++++
 4 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 mGAP/resources/queries/mGAP/sequenceDatasetsSource.query.xml

diff --git a/mGAP/resources/queries/mGAP/sequenceDatasetsSource.query.xml b/mGAP/resources/queries/mGAP/sequenceDatasetsSource.query.xml
new file mode 100644
index 000000000..3f4433200
--- /dev/null
+++ b/mGAP/resources/queries/mGAP/sequenceDatasetsSource.query.xml
@@ -0,0 +1,10 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="" tableDbType="TABLE">
+                <tableTitle>mGAP Sequence Datasets</tableTitle>
+
+            </table>
+        </tables>
+    </metadata>
+</query>
\ No newline at end of file
diff --git a/mGAP/resources/queries/mGAP/sequenceDatasetsSource.sql b/mGAP/resources/queries/mGAP/sequenceDatasetsSource.sql
index 0093ffe3f..e7484ab09 100644
--- a/mGAP/resources/queries/mGAP/sequenceDatasetsSource.sql
+++ b/mGAP/resources/queries/mGAP/sequenceDatasetsSource.sql
@@ -4,7 +4,8 @@ SELECT
   r.totalForwardReads as totalReads,
   r.sraRuns as sraAccession,
   r.subjectId as originalId,
-  (SELECT s.center FROM mgap.subjectssource s WHERE s.subjectName = am.externalAlias) as center
+  (SELECT s.center FROM mgap.subjectssource s WHERE s.subjectName = am.externalAlias) as center,
+  CASE WHEN (SELECT s.subjectName FROM mgap.subjectssource s WHERE s.subjectName = am.externalAlias) IS NULL THEN true ELSE false END as missingDemographics
 
 FROM sequenceanalysis.sequence_readsets r
 JOIN mgap.animalMapping am ON (r.subjectId = am.subjectname)
diff --git a/mGAP/resources/schemas/mgap.xml b/mGAP/resources/schemas/mgap.xml
index caed3fa92..ff56af1ae 100644
--- a/mGAP/resources/schemas/mgap.xml
+++ b/mGAP/resources/schemas/mgap.xml
@@ -291,6 +291,7 @@
                     <fkDbSchema>laboratory</fkDbSchema>
                     <fkTable>subjects</fkTable>
                     <fkColumnName>subjectname</fkColumnName>
+                    <fkDisplayColumnName useRawValue="true"/>
                 </fk>
             </column>
             <column columnName="sequenceType">
diff --git a/mGAP/resources/views/mgapDataDashboard.html b/mGAP/resources/views/mgapDataDashboard.html
index 5894d4de3..e43b7e7e5 100644
--- a/mGAP/resources/views/mgapDataDashboard.html
+++ b/mGAP/resources/views/mgapDataDashboard.html
@@ -81,6 +81,13 @@
                             queryName: 'subjectCaseMismatch',
                             'query.externalAlias~isnonblank': ''
                         })
+                    }, {
+                        name: 'Sequence Datasets Missing Demographics',
+                        url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {
+                            schemaName: 'mgap',
+                            queryName: 'sequenceDatasetsSource',
+                            'query.missingDemographics~eq': 'true'
+                        })
                     }]
                 },{
                     header: 'Prior Releases',
@@ -102,6 +109,12 @@
                             schemaName: 'mgap',
                             queryName: 'subjectsSource'
                         })
+                    }, {
+                        name: 'Sequence Datasets Synced to mGAP',
+                        url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {
+                            schemaName: 'mgap',
+                            queryName: 'sequenceDatasetsSource'
+                        })
                     }]
                 },{
                     header: 'Data and Processing',

From ccef459965a2344307ecdd589fb7c3bffc1d4859 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 19 Aug 2025 14:22:41 -0700
Subject: [PATCH 22/24] Add MCC column showing numAnimalsRequested

---
 mcc/resources/queries/mcc/requestScores/.qview.xml     | 1 +
 mcc/src/org/labkey/mcc/MccSchema.java                  | 1 +
 mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java | 8 ++++++++
 3 files changed, 10 insertions(+)

diff --git a/mcc/resources/queries/mcc/requestScores/.qview.xml b/mcc/resources/queries/mcc/requestScores/.qview.xml
index 3899ec233..f65fdeb51 100644
--- a/mcc/resources/queries/mcc/requestScores/.qview.xml
+++ b/mcc/resources/queries/mcc/requestScores/.qview.xml
@@ -7,6 +7,7 @@
         <column name="requestId/institutionname"/>
         <column name="requestId/fundingsource"/>
         <column name="requestId/grantnumber"/>
+        <column name="numAnimalsRequested"/>
         <column name="requestId/status"/>
         <column name="preliminaryScore"/>
         <column name="resourceAvailabilityAssessment"/>
diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java
index 03e4cd3ed..532fed97f 100644
--- a/mcc/src/org/labkey/mcc/MccSchema.java
+++ b/mcc/src/org/labkey/mcc/MccSchema.java
@@ -32,6 +32,7 @@ public class MccSchema
     public static final String TABLE_REQUEST_REVIEWS = "requestReviews";
     public static final String TABLE_REQUEST_SCORE = "requestScores";
     public static final String TABLE_CENSUS = "census";
+    public static final String TABLE_REQUEST_COHORTS = "requestcohorts";
 
     public static MccSchema getInstance()
     {
diff --git a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
index cf0f86072..59828fc3b 100644
--- a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
+++ b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
@@ -63,6 +63,14 @@ public boolean isSortable()
             ti.addColumn(newCol2);
         }
 
+        if (ti.getColumn("numAnimalsRequested") == null)
+        {
+            SQLFragment sql = new SQLFragment("(SELECT SUM(rc.numberofanimals) as expr FROM " + MccSchema.TABLE_REQUEST_COHORTS + " rc WHERE rc.requestId = " + ExprColumn.STR_TABLE_ALIAS + ".requestId)");
+            ExprColumn newCol = new ExprColumn(ti, "numAnimalsRequested", sql, JdbcType.INTEGER, ti.getColumn("requestId"));
+            newCol.setLabel("# Animals Requested");
+            ti.addColumn(newCol);
+        }
+
         if (ti.getColumn("transferIds") == null)
         {
             Container dataContainer = MccManager.get().getMCCContainer(ti.getContainer());

From bfaffc2f69faf13ae3d5250be765c04e6848adf9 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 19 Aug 2025 15:28:01 -0700
Subject: [PATCH 23/24] Sql fix

---
 mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
index 59828fc3b..42704019f 100644
--- a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
+++ b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
@@ -65,7 +65,7 @@ public boolean isSortable()
 
         if (ti.getColumn("numAnimalsRequested") == null)
         {
-            SQLFragment sql = new SQLFragment("(SELECT SUM(rc.numberofanimals) as expr FROM " + MccSchema.TABLE_REQUEST_COHORTS + " rc WHERE rc.requestId = " + ExprColumn.STR_TABLE_ALIAS + ".requestId)");
+            SQLFragment sql = new SQLFragment("(SELECT SUM(rc.numberofanimals) as expr FROM mcc." + MccSchema.TABLE_REQUEST_COHORTS + " rc WHERE rc.requestId = " + ExprColumn.STR_TABLE_ALIAS + ".requestId)");
             ExprColumn newCol = new ExprColumn(ti, "numAnimalsRequested", sql, JdbcType.INTEGER, ti.getColumn("requestId"));
             newCol.setLabel("# Animals Requested");
             ti.addColumn(newCol);

From 550b30f92504a4124f9cb6f11b027f69f3a17c98 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 19 Aug 2025 17:54:40 -0700
Subject: [PATCH 24/24] Add url to column

---
 mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
index 42704019f..44c9b5d02 100644
--- a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
+++ b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java
@@ -8,6 +8,7 @@
 import org.labkey.api.exp.api.StorageProvisioner;
 import org.labkey.api.ldk.table.AbstractTableCustomizer;
 import org.labkey.api.ldk.table.CustomPermissionsTable;
+import org.labkey.api.query.DetailsURL;
 import org.labkey.api.query.ExprColumn;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.study.Dataset;
@@ -68,6 +69,7 @@ public boolean isSortable()
             SQLFragment sql = new SQLFragment("(SELECT SUM(rc.numberofanimals) as expr FROM mcc." + MccSchema.TABLE_REQUEST_COHORTS + " rc WHERE rc.requestId = " + ExprColumn.STR_TABLE_ALIAS + ".requestId)");
             ExprColumn newCol = new ExprColumn(ti, "numAnimalsRequested", sql, JdbcType.INTEGER, ti.getColumn("requestId"));
             newCol.setLabel("# Animals Requested");
+            newCol.setURL(DetailsURL.fromString("/query/executeQuery.view?schemaName=mcc&query.queryName=" + MccSchema.TABLE_REQUEST_COHORTS + "&query.requestId~eq=${requestId}"));
             ti.addColumn(newCol);
         }