From 9e8f7fdd7f72e99e22e712b9da3fab7b6b8c983a Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 8 Sep 2025 06:07:50 -0700
Subject: [PATCH 01/40] Correct typo

---
 mGAP/resources/views/phenotypes.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mGAP/resources/views/phenotypes.html b/mGAP/resources/views/phenotypes.html
index e97e8e0c3..eb1f0e6ad 100644
--- a/mGAP/resources/views/phenotypes.html
+++ b/mGAP/resources/views/phenotypes.html
@@ -16,7 +16,7 @@
                 ['Nervous system','Vision','Coats-like retinopathy','','','Liu et al., 2015:25656754'],
                 ['Nervous system','Neurological','Batten disease','CLN7','c.769delA; p.Ile257LeufsTer36','McBride et al., 2018:30048804'],
                 ['Nervous system','Neurological','Krabbe disease','GALC','c.435_436delAC; p.Leu146fs','Luzi et al., 1997:9192853;Baskin et al., 1998:10090061', 'Hordeaux et al., 2022:35333110'],
-                ['Nervous system','Neurological','Pelizaiaeus-Merzbacher disease','PLP1','c.682 T > C; p.Cys228Arg','Sherman et al., 2021:34364975'],
+                ['Nervous system','Neurological','Pelizaeus-Merzbacher disease','PLP1','c.682 T > C; p.Cys228Arg','Sherman et al., 2021:34364975'],
                 ['Nervous system','Neurological','Epilepsy','','','Salinas et al., 2015:26290449;Akos Szabo et al., 2019:31592545'],
                 ['Nervous system','Psychiatric','Naltrexone response','OPRM1','c.77C>G; p.Pro26Arg','Vallender et al., 2010:20153935'],
                 ['Nervous system','Psychiatric','Anxiety ','5-HTT','5-HTTLPR','Spinelli  et al., 2012:22293001'],

From 37ab88963473cc32b4072c35dfad7585d3e9280f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 9 Sep 2025 10:34:35 -0700
Subject: [PATCH 02/40] Add MCC request field for shipping

---
 .../postgresql/mcc-20.018-20.019.sql          |   1 +
 .../dbscripts/sqlserver/mcc-20.018-20.019.sql |   1 +
 mcc/resources/schemas/mcc.xml                 |   4 +
 .../client/AnimalRequest/animal-request.tsx   | 557 ++++++++++--------
 .../client/AnimalRequest/components/values.ts |   1 +
 mcc/src/client/components/RequestUtils.tsx    |   2 +
 mcc/src/org/labkey/mcc/MccModule.java         |   2 +-
 .../org/labkey/test/tests/mcc/MccTest.java    |   1 +
 8 files changed, 334 insertions(+), 235 deletions(-)
 create mode 100644 mcc/resources/schemas/dbscripts/postgresql/mcc-20.018-20.019.sql
 create mode 100644 mcc/resources/schemas/dbscripts/sqlserver/mcc-20.018-20.019.sql

diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-20.018-20.019.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.018-20.019.sql
new file mode 100644
index 000000000..67506d33e
--- /dev/null
+++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.018-20.019.sql
@@ -0,0 +1 @@
+ALTER TABLE mcc.animalRequests ADD shippingAcknowledgement bool;
diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.018-20.019.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.018-20.019.sql
new file mode 100644
index 000000000..aa155ac90
--- /dev/null
+++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.018-20.019.sql
@@ -0,0 +1 @@
+ALTER TABLE mcc.animalRequests ADD shippingAcknowledgement bit;
diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml
index 86970ae21..0a0403e4f 100644
--- a/mcc/resources/schemas/mcc.xml
+++ b/mcc/resources/schemas/mcc.xml
@@ -373,6 +373,10 @@
                 <columnTitle>IACUC Protocol #</columnTitle>
                 <nullable>true</nullable>
             </column>
+            <column columnName="shippingAcknowledgement">
+                <columnTitle>Shipping Acknowledgement Entered?</columnTitle>
+                <nullable>true</nullable>
+            </column>
             <column columnName="comments">
                 <columnTitle>Other Comments</columnTitle>
                 <nullable>true</nullable>
diff --git a/mcc/src/client/AnimalRequest/animal-request.tsx b/mcc/src/client/AnimalRequest/animal-request.tsx
index e09acd148..fc79fd362 100644
--- a/mcc/src/client/AnimalRequest/animal-request.tsx
+++ b/mcc/src/client/AnimalRequest/animal-request.tsx
@@ -44,7 +44,8 @@ import {
     institutionTypeOptions,
     methodsProposedPlaceholder,
     signingOfficialTooltip,
-    terminalProceduresLabel
+    terminalProceduresLabel,
+    shippingAcknowledgementStatement
 } from './components/values';
 import AnimalCensus from './components/census';
 
@@ -330,6 +331,7 @@ export function AnimalRequest() {
                         "grantnumber" : data.get("funding-grant-number"),
                         "applicationduedate": data.get("funding-application-due-date"),
                         "comments": data.get("comments"),
+                        "shippingAcknowledgement": !!data.get("shippingAcknowledgement"),
                         "status": requestData.request.status,
                     }]
                 },
@@ -461,308 +463,395 @@ export function AnimalRequest() {
 
     return (
         <>
-        <form className="tw-w-full tw-max-w-4xl" onSubmit={handleSubmit} autoComplete="off" id={"animalRequestForm"} ref={formRef}>
-            <h3>Overview</h3>
+            <form className="tw-w-full tw-max-w-4xl" onSubmit={handleSubmit} autoComplete="off" id={'animalRequestForm'}
+                  ref={formRef}>
+                <h3>Overview</h3>
 
-            <Title text="1. Project Title*"/>
-            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <Input id="project-title" ariaLabel="Title" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} placeholder="Project Title" defaultValue={requestData.request.title}/>
-                </ErrorMessageHandler>
-            </div>
-
-            <Title text="2. Project Narrative*"/>
-            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <TextArea id="project-narrative" ariaLabel="Project Narrative" isSubmitting={isSubmitting} placeholder="Project Narrative" required={doEnforceRequiredFields()} defaultValue={requestData.request.narrative}/>
-                </ErrorMessageHandler>
-            </div>
-
-            <Title text="3. Research/Disease Focus*"/>
-            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <Input id="diseasefocus" ariaLabel="Research/Disease Focus" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} placeholder="What is the research area or disease focus of this project" defaultValue={requestData.request.diseasefocus}/>
-                </ErrorMessageHandler>
-            </div>
-
-            <Title text="4. How does the research relate to neuroscience?*"/>
-            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <TextArea id="neuroscience" ariaLabel="Connection to neuroscience" isSubmitting={isSubmitting} placeholder="How does the research relate to neuroscience" required={doEnforceRequiredFields()} defaultValue={requestData.request.neuroscience}/>
-                </ErrorMessageHandler>
-            </div>
-
-            <h3>General Information</h3>
-            <ErrorMessageHandler isSubmitting={isSubmitting}>
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                <Title text="1. Principal Investigator*"/>
-
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="investigator-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} placeholder="Last Name" defaultValue={requestData.request.lastname}/>
-                </div>
-
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="investigator-first-name" ariaLabel="First Name" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} placeholder="First Name" defaultValue={requestData.request.firstname}/>
-                </div>
-
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="investigator-middle-initial" isSubmitting={isSubmitting} required={false} placeholder="Middle Initial" defaultValue={requestData.request.middleinitial} maxLength="8"/>
-                </div>
-            </div>
-            </ErrorMessageHandler>
-
-            <ErrorMessageHandler isSubmitting={isSubmitting}>
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
-                    <Title text="2. Are you an early-stage investigator?&nbsp;"/>
-                    <Tooltip id="early-stage-investigator-helper"
-                       text={earlyInvestigatorTooltip}
-                    />
-                </div>
-
-                <div className="tw-w-full tw-px-3 tw-mt-6">
-                    <YesNoRadio id="is-early-stage-investigator" ariaLabel="Early Stage Investigator" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} defaultValue={requestData.request.earlystageinvestigator}/>
+                <Title text="1. Project Title*"/>
+                <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <Input id="project-title" ariaLabel="Title" isSubmitting={isSubmitting}
+                               required={doEnforceRequiredFields()} placeholder="Project Title"
+                               defaultValue={requestData.request.title}/>
+                    </ErrorMessageHandler>
                 </div>
-            </div>
-            </ErrorMessageHandler>
-
-            <ErrorMessageHandler isSubmitting={isSubmitting}>
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                <Title text="3. Affiliated research institution*"/>
 
+                <Title text="2. Project Narrative*"/>
                 <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="institution-name" ariaLabel="Institution Name" isSubmitting={isSubmitting} placeholder="Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutionname}/>
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <TextArea id="project-narrative" ariaLabel="Project Narrative" isSubmitting={isSubmitting}
+                                  placeholder="Project Narrative" required={doEnforceRequiredFields()}
+                                  defaultValue={requestData.request.narrative}/>
+                    </ErrorMessageHandler>
                 </div>
 
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="institution-city" ariaLabel="Institution City" isSubmitting={isSubmitting} placeholder="City" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutioncity}/>
+                <Title text="3. Research/Disease Focus*"/>
+                <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <Input id="diseasefocus" ariaLabel="Research/Disease Focus" isSubmitting={isSubmitting}
+                               required={doEnforceRequiredFields()}
+                               placeholder="What is the research area or disease focus of this project"
+                               defaultValue={requestData.request.diseasefocus}/>
+                    </ErrorMessageHandler>
                 </div>
 
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="institution-state" ariaLabel="Institution State" isSubmitting={isSubmitting} placeholder="State" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutionstate}/>
+                <Title text="4. How does the research relate to neuroscience?*"/>
+                <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <TextArea id="neuroscience" ariaLabel="Connection to neuroscience" isSubmitting={isSubmitting}
+                                  placeholder="How does the research relate to neuroscience"
+                                  required={doEnforceRequiredFields()} defaultValue={requestData.request.neuroscience}/>
+                    </ErrorMessageHandler>
                 </div>
 
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="institution-country" ariaLabel="Institution Country" isSubmitting={isSubmitting} placeholder="Country" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutioncountry}/>
-                </div>
+                <h3>General Information</h3>
+                <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                        <Title text="1. Principal Investigator*"/>
 
-                <Title text="4. Affiliated Research Institution Type*"/>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="investigator-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting}
+                                   required={doEnforceRequiredFields()} placeholder="Last Name"
+                                   defaultValue={requestData.request.lastname}/>
+                        </div>
 
-                <div className="tw-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Select id="institution-type" ariaLabel="Institution Type" isSubmitting={isSubmitting} placeholder="Type" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutiontype} options={institutionTypeOptions}/>
-                </div>
-            </div>
-            </ErrorMessageHandler>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="investigator-first-name" ariaLabel="First Name" isSubmitting={isSubmitting}
+                                   required={doEnforceRequiredFields()} placeholder="First Name"
+                                   defaultValue={requestData.request.firstname}/>
+                        </div>
 
-            <ErrorMessageHandler isSubmitting={isSubmitting}>
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
-                    <Title text="5. Institution Signing Official*&nbsp;"/>
-                    <Tooltip id="signing-official-helper"
-                         text={signingOfficialTooltip}
-                    />
-                </div>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="investigator-middle-initial" isSubmitting={isSubmitting} required={false}
+                                   placeholder="Middle Initial" defaultValue={requestData.request.middleinitial}
+                                   maxLength="8"/>
+                        </div>
+                    </div>
+                </ErrorMessageHandler>
 
+                <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                        <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
+                            <Title text="2. Are you an early-stage investigator?&nbsp;"/>
+                            <Tooltip id="early-stage-investigator-helper"
+                                     text={earlyInvestigatorTooltip}
+                            />
+                        </div>
 
-                <div className="tw-flex tw-flex-wrap tw-mt-6">
-                    <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="official-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting} placeholder="Last Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.officiallastname}/>
+                        <div className="tw-w-full tw-px-3 tw-mt-6">
+                            <YesNoRadio id="is-early-stage-investigator" ariaLabel="Early Stage Investigator"
+                                        isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                                        defaultValue={requestData.request.earlystageinvestigator}/>
+                        </div>
                     </div>
+                </ErrorMessageHandler>
 
-                    <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="official-first-name" ariaLabel="First Name" isSubmitting={isSubmitting} placeholder="First Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.officialfirstname}/>
-                    </div>
+                <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                        <Title text="3. Affiliated research institution*"/>
 
-                    <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="official-email" ariaLabel="Email Address" isSubmitting={isSubmitting} placeholder="Email Address" required={doEnforceRequiredFields()} defaultValue={requestData.request.officialemail}/>
-                    </div>
-                </div>
-            </div>
-            </ErrorMessageHandler>
+                        <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="institution-name" ariaLabel="Institution Name" isSubmitting={isSubmitting}
+                                   placeholder="Name" required={doEnforceRequiredFields()}
+                                   defaultValue={requestData.request.institutionname}/>
+                        </div>
 
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-10">
-                <Title text="6. Co-Investigators"/>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="institution-city" ariaLabel="Institution City" isSubmitting={isSubmitting}
+                                   placeholder="City" required={doEnforceRequiredFields()}
+                                   defaultValue={requestData.request.institutioncity}/>
+                        </div>
 
-                <CoInvestigators isSubmitting={isSubmitting} required={doEnforceRequiredFields()} coinvestigators={requestData.coinvestigators} onAddRecord={onAddInvestigator} onRemoveRecord={onRemoveCoInvestigator} />
-            </div>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="institution-state" ariaLabel="Institution State" isSubmitting={isSubmitting}
+                                   placeholder="State" required={doEnforceRequiredFields()}
+                                   defaultValue={requestData.request.institutionstate}/>
+                        </div>
 
-            <Title text="7. Existing or proposed funding source (select all that apply)"/>
-            {/* TODO: Make into checkbox group*/}
-            <Funding id="funding" isSubmitting={isSubmitting} defaultValue={requestData.request} required={doEnforceRequiredFields()}/>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="institution-country" ariaLabel="Institution Country" isSubmitting={isSubmitting}
+                                   placeholder="Country" required={doEnforceRequiredFields()}
+                                   defaultValue={requestData.request.institutioncountry}/>
+                        </div>
 
-            <h3>Institutional Animal Facilities and Capabilities</h3>
-            <div className="tw-w-full tw-px-3">
-                <Title text="1. Does your institution have existing NHP facilities?"/>
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                        <Select id="existing-nhp-facilities" ariaLabel="Existing NHP Facilities" isSubmitting={isSubmitting} options={existingNHPFacilityOptions} defaultValue={requestData.request.existingnhpfacilities} required={doEnforceRequiredFields()}/>
+                        <Title text="4. Affiliated Research Institution Type*"/>
+
+                        <div className="tw-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Select id="institution-type" ariaLabel="Institution Type" isSubmitting={isSubmitting}
+                                    placeholder="Type" required={doEnforceRequiredFields()}
+                                    defaultValue={requestData.request.institutiontype}
+                                    options={institutionTypeOptions}/>
+                        </div>
                     </div>
                 </ErrorMessageHandler>
 
-                <Title text="2. Does your institution have an existing marmoset colony?"/>
                 <ErrorMessageHandler isSubmitting={isSubmitting}>
                     <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                        <Select id="existing-marmoset-colony" ariaLabel="Existing Marmoset Colony" isSubmitting={isSubmitting} options={existingMarmosetColonyOptions} defaultValue={requestData.request.existingmarmosetcolony} required={doEnforceRequiredFields()}/>
+                        <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
+                            <Title text="5. Institution Signing Official*&nbsp;"/>
+                            <Tooltip id="signing-official-helper"
+                                     text={signingOfficialTooltip}
+                            />
+                        </div>
+
+
+                        <div className="tw-flex tw-flex-wrap tw-mt-6">
+                            <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="official-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting}
+                                       placeholder="Last Name" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.officiallastname}/>
+                            </div>
+
+                            <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="official-first-name" ariaLabel="First Name" isSubmitting={isSubmitting}
+                                       placeholder="First Name" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.officialfirstname}/>
+                            </div>
+
+                            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="official-email" ariaLabel="Email Address" isSubmitting={isSubmitting}
+                                       placeholder="Email Address" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.officialemail}/>
+                            </div>
+                        </div>
                     </div>
                 </ErrorMessageHandler>
 
-                <Title text="3. Do you plan to breed marmosets?"/>
-                <div className="tw-w-full tw-px-3 tw-mb-4">
-                    <AnimalBreeding id="animal-breeding" isSubmitting={isSubmitting} request={requestData.request} required={doEnforceRequiredFields()}/>
+                <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-10">
+                    <Title text="6. Co-Investigators"/>
+
+                    <CoInvestigators isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                                     coinvestigators={requestData.coinvestigators} onAddRecord={onAddInvestigator}
+                                     onRemoveRecord={onRemoveCoInvestigator}/>
                 </div>
-            </div>
 
-            <h3>Research Details</h3>
-
-            <div className="tw-flex tw-flex-wrap tw-mx-2">
-                <div className="tw-w-full tw-px-3 tw-mb-4">
-                    <Title text={"1. " + experimentalRationalePlaceholder}/>
-                    <Tooltip id="research-use-statement-helper"
-                       text={experimentalRationalePlaceholder}
-                    />
+                <Title text="7. Existing or proposed funding source (select all that apply)"/>
+                {/* TODO: Make into checkbox group*/}
+                <Funding id="funding" isSubmitting={isSubmitting} defaultValue={requestData.request}
+                         required={doEnforceRequiredFields()}/>
 
+                <h3>Institutional Animal Facilities and Capabilities</h3>
+                <div className="tw-w-full tw-px-3">
+                    <Title text="1. Does your institution have existing NHP facilities?"/>
                     <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <TextArea id="experiment-rationale" ariaLabel="Experimental rationale" isSubmitting={isSubmitting} placeholder={experimentalRationalePlaceholder} required={doEnforceRequiredFields()} defaultValue={requestData.request.experimentalrationale}/>
+                        <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                            <Select id="existing-nhp-facilities" ariaLabel="Existing NHP Facilities"
+                                    isSubmitting={isSubmitting} options={existingNHPFacilityOptions}
+                                    defaultValue={requestData.request.existingnhpfacilities}
+                                    required={doEnforceRequiredFields()}/>
+                        </div>
                     </ErrorMessageHandler>
-                </div>
-
-                <Title text="2. Animal Cohorts"/>
-                <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-6">
-                    <AnimalCohorts isSubmitting={isSubmitting} cohorts={requestData.cohorts} required={doEnforceRequiredFields()} onAddCohort={onAddCohort} onRemoveCohort={onRemoveCohort}/>
-                </div>
 
-                <Title text={"3. " + methodsProposedPlaceholder}/>
-                <div className="tw-w-full tw-px-3 tw-mb-6">
+                    <Title text="2. Does your institution have an existing marmoset colony?"/>
                     <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <TextArea id="methods-proposed" ariaLabel="Methods Proposed" isSubmitting={isSubmitting} placeholder={methodsProposedPlaceholder} required={doEnforceRequiredFields()} defaultValue={requestData.request.methodsproposed}/>
-                    </div>
+                        <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                            <Select id="existing-marmoset-colony" ariaLabel="Existing Marmoset Colony"
+                                    isSubmitting={isSubmitting} options={existingMarmosetColonyOptions}
+                                    defaultValue={requestData.request.existingmarmosetcolony}
+                                    required={doEnforceRequiredFields()}/>
+                        </div>
                     </ErrorMessageHandler>
+
+                    <Title text="3. Do you plan to breed marmosets?"/>
+                    <div className="tw-w-full tw-px-3 tw-mb-4">
+                        <AnimalBreeding id="animal-breeding" isSubmitting={isSubmitting} request={requestData.request}
+                                        required={doEnforceRequiredFields()}/>
+                    </div>
                 </div>
 
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                        <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
-                            <Title text={"4. " + terminalProceduresLabel}/>
-                        </div>
+                <h3>Research Details</h3>
+
+                <div className="tw-flex tw-flex-wrap tw-mx-2">
+                    <div className="tw-w-full tw-px-3 tw-mb-4">
+                        <Title text={'1. ' + experimentalRationalePlaceholder}/>
+                        <Tooltip id="research-use-statement-helper"
+                                 text={experimentalRationalePlaceholder}
+                        />
+
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <TextArea id="experiment-rationale" ariaLabel="Experimental rationale"
+                                      isSubmitting={isSubmitting} placeholder={experimentalRationalePlaceholder}
+                                      required={doEnforceRequiredFields()}
+                                      defaultValue={requestData.request.experimentalrationale}/>
+                        </ErrorMessageHandler>
+                    </div>
 
-                        <div className="tw-w-full tw-px-3 tw-mt-6">
-                            <YesNoRadio id="is-terminalprocedures" ariaLabel="Terminal procedures" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} defaultValue={requestData.request.terminalprocedures}/>
-                        </div>
+                    <Title text="2. Animal Cohorts"/>
+                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-6">
+                        <AnimalCohorts isSubmitting={isSubmitting} cohorts={requestData.cohorts}
+                                       required={doEnforceRequiredFields()} onAddCohort={onAddCohort}
+                                       onRemoveCohort={onRemoveCohort}/>
                     </div>
-                </ErrorMessageHandler>
 
-                <Title text={"5. " + collaborationsPlaceholder}/>
-                <div className="tw-w-full tw-px-3 tw-mb-6">
-                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <Title text={'3. ' + methodsProposedPlaceholder}/>
                     <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <TextArea id="collaborations" ariaLabel="Collaborations" isSubmitting={isSubmitting} placeholder={collaborationsPlaceholder} required={doEnforceRequiredFields()} defaultValue={requestData.request.collaborations}/>
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <div className="tw-w-full tw-px-3 tw-mb-6">
+                                <TextArea id="methods-proposed" ariaLabel="Methods Proposed" isSubmitting={isSubmitting}
+                                          placeholder={methodsProposedPlaceholder} required={doEnforceRequiredFields()}
+                                          defaultValue={requestData.request.methodsproposed}/>
+                            </div>
+                        </ErrorMessageHandler>
                     </div>
-                    </ErrorMessageHandler>
-                </div>
 
-                <Title text={"6. " + animalWellfarePlaceholder}/>
-                <div className="tw-w-full tw-px-3 tw-mb-6">
                     <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                            <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
+                                <Title text={'4. ' + terminalProceduresLabel}/>
+                            </div>
+
+                            <div className="tw-w-full tw-px-3 tw-mt-6">
+                                <YesNoRadio id="is-terminalprocedures" ariaLabel="Terminal procedures"
+                                            isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                                            defaultValue={requestData.request.terminalprocedures}/>
+                            </div>
+                        </div>
+                    </ErrorMessageHandler>
+
+                    <Title text={'5. ' + collaborationsPlaceholder}/>
                     <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <TextArea id="animal-welfare" ariaLabel="Animal Welfare" isSubmitting={isSubmitting} placeholder={animalWellfarePlaceholder} required={doEnforceRequiredFields()} defaultValue={requestData.request.animalwelfare}/>
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <div className="tw-w-full tw-px-3 tw-mb-6">
+                                <TextArea id="collaborations" ariaLabel="Collaborations" isSubmitting={isSubmitting}
+                                          placeholder={collaborationsPlaceholder} required={doEnforceRequiredFields()}
+                                          defaultValue={requestData.request.collaborations}/>
+                            </div>
+                        </ErrorMessageHandler>
                     </div>
-                    </ErrorMessageHandler>
 
-                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <Title text={'6. ' + animalWellfarePlaceholder}/>
                     <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <input type="checkbox" name="certify" id="certify" aria-label="Certify" className={(isSubmitting ? "custom-invalid" : "")} required={doEnforceRequiredFields()} defaultChecked={requestData.request.certify}/>
-                        <label className="tw-text-gray-700 ml-1">{certificationLabel}</label>
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <div className="tw-w-full tw-px-3 tw-mb-6">
+                                <TextArea id="animal-welfare" ariaLabel="Animal Welfare" isSubmitting={isSubmitting}
+                                          placeholder={animalWellfarePlaceholder} required={doEnforceRequiredFields()}
+                                          defaultValue={requestData.request.animalwelfare}/>
+                            </div>
+                        </ErrorMessageHandler>
+
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <div className="tw-w-full tw-px-3 tw-mb-6">
+                                <input type="checkbox" name="certify" id="certify" aria-label="Certify"
+                                       className={(isSubmitting ? 'custom-invalid' : '')}
+                                       required={doEnforceRequiredFields()}
+                                       defaultChecked={requestData.request.certify}/>
+                                <label className="tw-text-gray-700 ml-1">{certificationLabel}</label>
+                            </div>
+                        </ErrorMessageHandler>
                     </div>
+
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                            <Title text="7. Attending veterinarian"/>
+
+                            <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="vet-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting}
+                                       placeholder="Last Name" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.vetlastname}/>
+                            </div>
+
+                            <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="vet-first-name" ariaLabel="First Name" isSubmitting={isSubmitting}
+                                       placeholder="First Name" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.vetfirstname}/>
+                            </div>
+
+                            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="vet-email" ariaLabel="Email" isSubmitting={isSubmitting}
+                                       placeholder="Email Address" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.vetemail}/>
+                            </div>
+                        </div>
                     </ErrorMessageHandler>
                 </div>
 
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                    <Title text="7. Attending veterinarian"/>
-
-                    <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="vet-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting} placeholder="Last Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.vetlastname}/>
-                    </div>
-
-                    <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="vet-first-name" ariaLabel="First Name" isSubmitting={isSubmitting} placeholder="First Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.vetfirstname}/>
-                    </div>
+                <IACUCProtocol id="iacuc" isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                               request={requestData.request}/>
 
-                    <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="vet-email" ariaLabel="Email" isSubmitting={isSubmitting} placeholder="Email Address" required={doEnforceRequiredFields()} defaultValue={requestData.request.vetemail}/>
-                    </div>
+                <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
+                    <Title text="9. Will participate in the MCC Census?&nbsp;"/>
+                    <Tooltip id="census-helper" text={censusToolTip}/>
+                </div>
+                <div className="tw-w-full tw-px-3 tw-mt-3">
+                    <AnimalCensus id="census" isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                                  request={requestData.request}/>
                 </div>
-                </ErrorMessageHandler>
-            </div>
-
-            <IACUCProtocol id="iacuc" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} request={requestData.request}/>
 
-            <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
-                <Title text="9. Will participate in the MCC Census?&nbsp;"/>
-                <Tooltip id="census-helper" text={censusToolTip}/>
-            </div>
-            <div className="tw-w-full tw-px-3 tw-mt-3">
-                <AnimalCensus id="census" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} request={requestData.request}/>
-            </div>
+                <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-4">
+                    <Title text="10. Shipment Acknowledgement&nbsp;"/>
+                </div>
 
-            <Title text={"10. " + commentsPlaceholder}/>
-            <div className="tw-w-full tw-px-3 tw-mb-6">
                 <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <TextArea id="comments" ariaLabel="Comments" isSubmitting={isSubmitting} placeholder={commentsPlaceholder} required={false} defaultValue={requestData.request.comments}/>
+                    <div className="tw-w-full tw-px-6 tw-mb-6">
+                        {shippingAcknowledgementStatement}
+                        <p />
+                        <input type="checkbox" name="shippingAcknowledgement" id="shippingAcknowledgement" aria-label="Shipping Acknowledgement"
+                               className={(isSubmitting ? 'custom-invalid' : '')}
+                               required={doEnforceRequiredFields()}
+                               defaultChecked={requestData.request.shippingAcknowledgement}/>
+                        <label className="tw-text-gray-700 ml-1">I acknowledge this statement</label>
                     </div>
                 </ErrorMessageHandler>
-            </div>
 
-            <div className="tw-flex tw-flex-wrap tw-mx-2">
-                <Title text="Request Status: "/>{requestData.request.status}
-            </div>
+                <Title text={'11. ' + commentsPlaceholder}/>
+                <div className="tw-w-full tw-px-3 tw-mb-6">
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <div className="tw-w-full tw-px-3 tw-mb-6">
+                            <TextArea id="comments" ariaLabel="Comments" isSubmitting={isSubmitting}
+                                      placeholder={commentsPlaceholder} required={false}
+                                      defaultValue={requestData.request.comments}/>
+                        </div>
+                    </ErrorMessageHandler>
+                </div>
 
-            <div className="tw-flex tw-flex-wrap tw-mx-2">
-                <Button baseColor="red" marginLeft="auto" text="Cancel" onClick={(e) => {
-                    e.preventDefault()
+                <div className="tw-flex tw-flex-wrap tw-mx-2">
+                    <Title text="Request Status: "/>{requestData.request.status}
+                </div>
 
-                    if (confirm("You are about to leave this page.")) {
-                        window.location.href = ActionURL.buildURL('mcc', 'mccRequests.view')
-                    }
-                }} />
+                <div className="tw-flex tw-flex-wrap tw-mx-2">
+                    <Button baseColor="red" marginLeft="auto" text="Cancel" onClick={(e) => {
+                        e.preventDefault();
 
-                <Button onClick={(e) => {
-                    handleSubmitButton(e, false);
-                 }} text={getSaveButtonText()} display={hasEditPermission()}/>
+                        if (confirm('You are about to leave this page.')) {
+                            window.location.href = ActionURL.buildURL('mcc', 'mccRequests.view');
+                        }
+                    }}/>
 
-                <Button onClick={(e) => {
-                    handleSubmitButton(e, true);
-                 }} text={getSubmitButtonText()} display={hasEditPermission()}/>
+                    <Button onClick={(e) => {
+                        handleSubmitButton(e, false);
+                    }} text={getSaveButtonText()} display={hasEditPermission()}/>
 
-                <Button onClick={(e) => {
-                    e.preventDefault()
-                    setShowWithdrawDialog(true)
-                }} text={"Withdraw"} display={shouldShowWithdraw()}/>
-            </div>
-        </form>
-
-        <SavingOverlay display={displayOverlay} />
-
-        <Dialog open={showWithdrawDialog}>
-            <DialogTitle>Withdraw Request</DialogTitle>
-            <DialogContent>
-                <DialogContentText>Please enter a reason for withdrawing this request</DialogContentText>
-                <TextareaAutosize
-                    minRows={4}
-                    id="withdrawReason"
-                    required={true}
-                    autoFocus={true}
-                    defaultValue={withdrawReasonText}
-                    form={"animalRequestForm"}
-                    onChange={(e) => setWithdrawReasonText(e.target.value)}
-                />
-            </DialogContent>
-            <DialogActions>
-                <Box mr="5px">
                     <Button onClick={(e) => {
-                        if (!withdrawReasonText) {
+                        handleSubmitButton(e, true);
+                    }} text={getSubmitButtonText()} display={hasEditPermission()}/>
+
+                    <Button onClick={(e) => {
+                        e.preventDefault();
+                        setShowWithdrawDialog(true);
+                    }} text={'Withdraw'} display={shouldShowWithdraw()}/>
+                </div>
+            </form>
+
+            <SavingOverlay display={displayOverlay}/>
+
+            <Dialog open={showWithdrawDialog}>
+                <DialogTitle>Withdraw Request</DialogTitle>
+                <DialogContent>
+                    <DialogContentText>Please enter a reason for withdrawing this request</DialogContentText>
+                    <TextareaAutosize
+                        minRows={4}
+                        id="withdrawReason"
+                        required={true}
+                        autoFocus={true}
+                        defaultValue={withdrawReasonText}
+                        form={'animalRequestForm'}
+                        onChange={(e) => setWithdrawReasonText(e.target.value)}
+                    />
+                </DialogContent>
+                <DialogActions>
+                    <Box mr="5px">
+                        <Button onClick={(e) => {
+                            if (!withdrawReasonText) {
                             alert("Must enter the reason")
                             return
                         }
diff --git a/mcc/src/client/AnimalRequest/components/values.ts b/mcc/src/client/AnimalRequest/components/values.ts
index 055ea750a..afb771376 100644
--- a/mcc/src/client/AnimalRequest/components/values.ts
+++ b/mcc/src/client/AnimalRequest/components/values.ts
@@ -14,6 +14,7 @@ export const animalWellfarePlaceholder = "Animal welfare (proposed care and use)
 export const censusReasonPlaceholder = "Reason for not participating"
 
 export const certificationLabel = "I certify and I have obtained approval for this study from my institution."
+export const shippingAcknowledgementStatement = "I will be ready to receive animals within 60 days of approval, provided that they are available from a breeding center.  I understand that failure to do so will result in per diem charges billed to me."
 export const terminalProceduresLabel = "Includes terminal procedures?"
 
 export const fundingSourceOptions = [
diff --git a/mcc/src/client/components/RequestUtils.tsx b/mcc/src/client/components/RequestUtils.tsx
index b25c4a8e6..4bca1478b 100644
--- a/mcc/src/client/components/RequestUtils.tsx
+++ b/mcc/src/client/components/RequestUtils.tsx
@@ -48,6 +48,7 @@ export class AnimalRequestProps {
     vetlastname: string;
     vetemail: string;
     vetfirstname: string;
+    shippingAcknowledgement: boolean;
     objectid: string;
     comments: string;
 }
@@ -145,6 +146,7 @@ export async function queryRequestInformation(requestId, handleFailure) {
                 "iacucprotocol",
                 "grantnumber",
                 "applicationduedate",
+                "shippingAcknowledgement",
                 "comments",
                 "status"
             ],
diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java
index c52176612..11292760b 100644
--- a/mcc/src/org/labkey/mcc/MccModule.java
+++ b/mcc/src/org/labkey/mcc/MccModule.java
@@ -77,7 +77,7 @@ public String getName()
     @Override
     public @Nullable Double getSchemaVersion()
     {
-        return 20.018;
+        return 20.019;
     }
 
     @Override
diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java
index 300e78234..9c5caf4f7 100644
--- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java
+++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java
@@ -463,6 +463,7 @@ else if ("radio".equals(inputType))
             new FormElement("existing-nhp-facilities", "existingnhpfacilities", "Existing NHP facilities").select("existing"),
             new FormElement("animal-welfare", "animalwelfare", "welfare").inputType("textarea"),
             new FormElement("certify", "certify", true).checkBox(),
+            new FormElement("shippingAcknowledgement", "shippingAcknowledgement", true).checkBox(),
             new FormElement("vet-last-name", "vetlastname", "vet last name"),
             new FormElement("vet-first-name", "vetfirstname", "vet first name"),
             new FormElement("vet-email", "vetemail", "vet@email.com"),

From 24fe62d339c1cdd45f3e79e9b2994dd1229ea29f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 12 Sep 2025 10:07:35 -0700
Subject: [PATCH 03/40] Minor code cleanup

---
 SivStudies/resources/queries/study/samples/.qview.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/SivStudies/resources/queries/study/samples/.qview.xml b/SivStudies/resources/queries/study/samples/.qview.xml
index 830bf2440..8581fec3c 100644
--- a/SivStudies/resources/queries/study/samples/.qview.xml
+++ b/SivStudies/resources/queries/study/samples/.qview.xml
@@ -2,6 +2,7 @@
     <columns>
         <column name="Id"/>
         <column name="date"/>
+        <column name="timePostSivChallenge/daysPostInfection"/>
         <column name="sampleType"/>
         <column name="quantity"/>
         <column name="quantity_units"/>

From 01a05d60890ad3a581cb985ce0228f2f7106a574 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 15 Sep 2025 15:01:31 -0700
Subject: [PATCH 04/40] Improve sample query

---
 IDR/resources/queries/bimber_data/idrSampleSource.sql | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/IDR/resources/queries/bimber_data/idrSampleSource.sql b/IDR/resources/queries/bimber_data/idrSampleSource.sql
index 0a38d53a6..e9a7baa68 100644
--- a/IDR/resources/queries/bimber_data/idrSampleSource.sql
+++ b/IDR/resources/queries/bimber_data/idrSampleSource.sql
@@ -18,7 +18,12 @@ SELECT
 Rh as Id,
 ID as sampleid,
 SampleDate as date,
-Tissue as sampleType,
+CASE
+    WHEN (Tissue IS NOT NULL AND Tissue != '') AND (SampleType IS NOT NULL AND SampleType != '') THEN (SampleType || ' / ' || Tissue)
+    WHEN (Tissue IS NOT NULL AND Tissue = '') THEN Tissue
+    WHEN (SampleType IS NOT NULL AND Tissue = '') THEN SampleType
+    ELSE NULL
+END AS sampleType,
 null as quantity,
 
 'Hansen/IDR' as dataSource

From 9edb28bdaf08b4c1136404f7c2e9188d9cd5a6e3 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 16 Sep 2025 06:48:57 -0700
Subject: [PATCH 05/40] Add CSP exception

---
 mGAP/src/org/labkey/mgap/mGAPModule.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mGAP/src/org/labkey/mgap/mGAPModule.java b/mGAP/src/org/labkey/mgap/mGAPModule.java
index 9375e0a70..3799f84c1 100644
--- a/mGAP/src/org/labkey/mgap/mGAPModule.java
+++ b/mGAP/src/org/labkey/mgap/mGAPModule.java
@@ -111,6 +111,7 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext)
         ContentSecurityPolicyFilter.registerAllowedSources(this.getClass().getName(), Directive.Connection, "https://code.jquery.com", "https://*.fontawesome.com");
         ContentSecurityPolicyFilter.registerAllowedSources(this.getClass().getName(), Directive.Style, "https://code.jquery.com", "https://www.gstatic.com");
         ContentSecurityPolicyFilter.registerAllowedSources(this.getClass().getName(), Directive.Font, "https://*.fontawesome.com");
+        ContentSecurityPolicyFilter.registerAllowedSources(this.getClass().getName(), Directive.Connection, "https://oss.maxcdn.com");
 
         new PipelineStartup();
     }

From 583865c59fd2c8f6fd740a8f23b488046a1a4106 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 17 Sep 2025 11:42:05 -0700
Subject: [PATCH 06/40] Update outcome SQL

---
 IDR/resources/queries/bimber_data/idrOutcomeSource.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql
index 7621857ce..d254000ca 100644
--- a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql
+++ b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql
@@ -6,6 +6,7 @@ cohortStart as date,
 CASE
     WHEN contprog = 'C' THEN 'Controller'
     WHEN contprog = 'P' THEN 'Progressor'
+    ELSE contprog
 END as outcome,
 
 'Hansen/IDR' as dataSource

From c2a6067b1efa410b73bd348327a864909d3f879b Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 09:41:14 -0700
Subject: [PATCH 07/40] Add comments columns

---
 .../study/additionalDatatypes.query.xml       |  3 +++
 .../study/additionalDatatypes/.qview.xml      |  1 +
 .../resources/queries/study/flow.query.xml    |  2 +-
 .../resources/queries/study/flow/.qview.xml   |  1 +
 .../queries/study/genetics.query.xml          |  4 +++
 .../queries/study/genetics/.qview.xml         |  1 +
 .../queries/study/immunizations.query.xml     |  4 +++
 .../queries/study/immunizations/.qview.xml    |  1 +
 .../resources/queries/study/labwork.query.xml |  4 +++
 .../queries/study/labwork/.qview.xml          |  1 +
 .../queries/study/procedures.query.xml        |  4 +++
 .../queries/study/procedures/.qview.xml       |  1 +
 .../resources/queries/study/samples.query.xml |  4 +++
 .../queries/study/samples/.qview.xml          |  1 +
 .../queries/study/treatments.query.xml        |  4 +++
 .../queries/study/treatments/.qview.xml       |  1 +
 .../queries/study/viralLoads/.qview.xml       |  1 +
 .../queries/study/viralloads.query.xml        |  4 +++
 .../study/datasets/datasets_metadata.xml      | 26 ++++++++++++++++++-
 19 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/SivStudies/resources/queries/study/additionalDatatypes.query.xml b/SivStudies/resources/queries/study/additionalDatatypes.query.xml
index 1b446dfde..45261263a 100644
--- a/SivStudies/resources/queries/study/additionalDatatypes.query.xml
+++ b/SivStudies/resources/queries/study/additionalDatatypes.query.xml
@@ -18,6 +18,9 @@
                     <column columnName="description">
                         <columnTitle>Description</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                    </column>
                 </columns>
             </table>
         </tables>
diff --git a/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml b/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml
index 612c76c8a..46110c9f0 100644
--- a/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml
+++ b/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml
@@ -5,6 +5,7 @@
         <column name="timePostSivChallenge/daysPostInfection"/>
         <column name="category"/>
         <column name="description"/>
+        <column name="comments"/>
         <column name="dataSource"/>
     </columns>
     <sorts>
diff --git a/SivStudies/resources/queries/study/flow.query.xml b/SivStudies/resources/queries/study/flow.query.xml
index 63f49ed2f..63a7946a4 100644
--- a/SivStudies/resources/queries/study/flow.query.xml
+++ b/SivStudies/resources/queries/study/flow.query.xml
@@ -24,7 +24,7 @@
                     <column columnName="units">
                         <columnTitle>Units</columnTitle>
                     </column>
-                    <column columnName="comment">
+                    <column columnName="comments">
                         <columnTitle>Comments</columnTitle>
                     </column>
                     <column columnName="dataSource">
diff --git a/SivStudies/resources/queries/study/flow/.qview.xml b/SivStudies/resources/queries/study/flow/.qview.xml
index c00809cd5..7c9abe0b8 100644
--- a/SivStudies/resources/queries/study/flow/.qview.xml
+++ b/SivStudies/resources/queries/study/flow/.qview.xml
@@ -7,6 +7,7 @@
         <column name="population"/>
         <column name="result"/>
         <column name="units"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
     </columns>
diff --git a/SivStudies/resources/queries/study/genetics.query.xml b/SivStudies/resources/queries/study/genetics.query.xml
index 937945710..334a603e5 100644
--- a/SivStudies/resources/queries/study/genetics.query.xml
+++ b/SivStudies/resources/queries/study/genetics.query.xml
@@ -23,6 +23,10 @@
                     <column columnName="score">
                         <columnTitle>Score</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/genetics/.qview.xml b/SivStudies/resources/queries/study/genetics/.qview.xml
index bb737b813..0a9321497 100644
--- a/SivStudies/resources/queries/study/genetics/.qview.xml
+++ b/SivStudies/resources/queries/study/genetics/.qview.xml
@@ -7,6 +7,7 @@
         <column name="marker"/>
         <column name="result"/>
         <column name="score"/>
+        <column name="comments"/>
         <column name="dataSource"/>
     </columns>
     <sorts>
diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml
index 8e2c1528d..7e1a7a603 100644
--- a/SivStudies/resources/queries/study/immunizations.query.xml
+++ b/SivStudies/resources/queries/study/immunizations.query.xml
@@ -27,6 +27,10 @@
                     <column columnName="reason">
                         <columnTitle>Reason</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/immunizations/.qview.xml b/SivStudies/resources/queries/study/immunizations/.qview.xml
index 8264aae36..575150b0e 100644
--- a/SivStudies/resources/queries/study/immunizations/.qview.xml
+++ b/SivStudies/resources/queries/study/immunizations/.qview.xml
@@ -7,6 +7,7 @@
         <column name="route"/>
         <column name="quantity"/>
         <column name="quantity_units"/>
+        <column name="comments"/>
         <column name="dataSource"/>
     </columns>
     <sorts>
diff --git a/SivStudies/resources/queries/study/labwork.query.xml b/SivStudies/resources/queries/study/labwork.query.xml
index c0808ad17..1cb7b4022 100644
--- a/SivStudies/resources/queries/study/labwork.query.xml
+++ b/SivStudies/resources/queries/study/labwork.query.xml
@@ -34,6 +34,10 @@
                     <column columnName="method">
                         <columnTitle>Method</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/labwork/.qview.xml b/SivStudies/resources/queries/study/labwork/.qview.xml
index 553672473..800012954 100644
--- a/SivStudies/resources/queries/study/labwork/.qview.xml
+++ b/SivStudies/resources/queries/study/labwork/.qview.xml
@@ -7,6 +7,7 @@
         <column name="result"/>
         <column name="units"/>
         <column name="method"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
         <column name="ageAtTime/AgeAtTime"/>
diff --git a/SivStudies/resources/queries/study/procedures.query.xml b/SivStudies/resources/queries/study/procedures.query.xml
index eaa6a9606..1f5446fe1 100644
--- a/SivStudies/resources/queries/study/procedures.query.xml
+++ b/SivStudies/resources/queries/study/procedures.query.xml
@@ -15,6 +15,10 @@
                     <column columnName="procedure">
                         <columnTitle>Procedure</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/procedures/.qview.xml b/SivStudies/resources/queries/study/procedures/.qview.xml
index 01c43bfd2..a7fb29c4a 100644
--- a/SivStudies/resources/queries/study/procedures/.qview.xml
+++ b/SivStudies/resources/queries/study/procedures/.qview.xml
@@ -4,6 +4,7 @@
         <column name="date"/>
         <column name="category"/>
         <column name="procedure"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
     </columns>
diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml
index a65518efc..3aaacd32e 100644
--- a/SivStudies/resources/queries/study/samples.query.xml
+++ b/SivStudies/resources/queries/study/samples.query.xml
@@ -23,6 +23,10 @@
                     <column columnName="quantity_units">
                         <columnTitle>Quantity Units</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/samples/.qview.xml b/SivStudies/resources/queries/study/samples/.qview.xml
index 8581fec3c..0e70ce34d 100644
--- a/SivStudies/resources/queries/study/samples/.qview.xml
+++ b/SivStudies/resources/queries/study/samples/.qview.xml
@@ -7,6 +7,7 @@
         <column name="quantity"/>
         <column name="quantity_units"/>
         <column name="sampleId"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
         <column name="ageAtTime/AgeAtTime"/>
diff --git a/SivStudies/resources/queries/study/treatments.query.xml b/SivStudies/resources/queries/study/treatments.query.xml
index 52a3ba764..b053767c0 100644
--- a/SivStudies/resources/queries/study/treatments.query.xml
+++ b/SivStudies/resources/queries/study/treatments.query.xml
@@ -65,6 +65,10 @@
                         <isHidden>true</isHidden>
                         <shownInDetailsView>false</shownInDetailsView>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/treatments/.qview.xml b/SivStudies/resources/queries/study/treatments/.qview.xml
index d9b12386c..070ec43c1 100644
--- a/SivStudies/resources/queries/study/treatments/.qview.xml
+++ b/SivStudies/resources/queries/study/treatments/.qview.xml
@@ -8,6 +8,7 @@
         <column name="route"/>
         <column name="amount"/>
         <column name="amount_units"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
     </columns>
diff --git a/SivStudies/resources/queries/study/viralLoads/.qview.xml b/SivStudies/resources/queries/study/viralLoads/.qview.xml
index ef089d8f7..3be12098f 100644
--- a/SivStudies/resources/queries/study/viralLoads/.qview.xml
+++ b/SivStudies/resources/queries/study/viralLoads/.qview.xml
@@ -9,6 +9,7 @@
         <column name="result"/>
         <column name="units"/>
         <column name="lod"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="artInformation/daysPostArtInitiation"/>
         <column name="artInformation/daysPostArtRelease"/>
diff --git a/SivStudies/resources/queries/study/viralloads.query.xml b/SivStudies/resources/queries/study/viralloads.query.xml
index 4e9e9af40..a53069a24 100644
--- a/SivStudies/resources/queries/study/viralloads.query.xml
+++ b/SivStudies/resources/queries/study/viralloads.query.xml
@@ -39,6 +39,10 @@
                         <isHidden>true</isHidden>
                         <shownInDetailsView>false</shownInDetailsView>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 5eb90a75a..181994629 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -87,6 +87,9 @@
             <column columnName="reason">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Medications/Treatments</tableTitle>
     </table>
@@ -126,6 +129,9 @@
             <column columnName="reason">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Immunizations</tableTitle>
     </table>
@@ -199,6 +205,9 @@
             <column columnName="qualresult">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Viral Loads</tableTitle>
     </table>
@@ -240,6 +249,9 @@
             <column columnName="method">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Lab Results</tableTitle>
     </table>
@@ -360,6 +372,9 @@
             <column columnName="quantity_units">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Samples</tableTitle>
     </table>
@@ -395,6 +410,9 @@
             <column columnName="score">
                 <datatype>double</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Genetic Data</tableTitle>
     </table>
@@ -421,6 +439,9 @@
             <column columnName="procedure">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Procedures</tableTitle>
     </table>
@@ -447,6 +468,9 @@
             <column columnName="description">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Additional Datatypes</tableTitle>
     </table>
@@ -482,7 +506,7 @@
             <column columnName="units">
                 <datatype>varchar</datatype>
             </column>
-            <column columnName="comment">
+            <column columnName="comments">
                 <datatype>varchar</datatype>
             </column>
         </columns>

From ff1baa5412e12f52fa70645a3abeeeea85ad5a8c Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 10:37:37 -0700
Subject: [PATCH 08/40] Add ETL to automatically create subjects for SIV
 studies

---
 SivStudies/resources/etls/idr-subjects.xml    | 15 +++
 .../sivstudies/etl/AddMissingIdrSubjects.java | 98 +++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 SivStudies/resources/etls/idr-subjects.xml
 create mode 100644 SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java

diff --git a/SivStudies/resources/etls/idr-subjects.xml b/SivStudies/resources/etls/idr-subjects.xml
new file mode 100644
index 000000000..fd084bead
--- /dev/null
+++ b/SivStudies/resources/etls/idr-subjects.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml">
+    <name>Hansen/IDR Subjects</name>
+    <description>Hansen/IDR Subjects</description>
+    <transforms>
+        <transform id="subjects" type="TaskRefTransformStep">
+            <taskref ref="org.labkey.sivstudies.etl.AddMissingIdrSubjects">
+
+            </taskref>
+        </transform>
+    </transforms>
+    <schedule>
+        <cron expression="0 15 20 * * ?"/>
+    </schedule>
+</etl>
diff --git a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
new file mode 100644
index 000000000..c0873e312
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
@@ -0,0 +1,98 @@
+package org.labkey.sivstudies.etl;
+
+import org.apache.xmlbeans.XmlException;
+import org.jetbrains.annotations.NotNull;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.di.TaskRefTask;
+import org.labkey.api.pipeline.PipelineJob;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.pipeline.RecordedActionSet;
+import org.labkey.api.query.BatchValidationException;
+import org.labkey.api.query.DuplicateKeyException;
+import org.labkey.api.query.QueryService;
+import org.labkey.api.query.QueryUpdateServiceException;
+import org.labkey.api.query.UserSchema;
+import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.writer.ContainerUser;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class AddMissingIdrSubjects implements TaskRefTask
+{
+    protected ContainerUser _containerUser;
+
+    @Override
+    public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJobException
+    {
+        // Find existing IDs:
+        UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study");
+        if (us == null)
+        {
+            throw new PipelineJobException("Missing study schema");
+        }
+
+        List<String> existingIds = new ArrayList<>(new TableSelector(us.getTable("demographics"), PageFlowUtil.set("Id"), null, null).getArrayList(String.class));
+
+        // Source IDs:
+        Container sourceContainer = ContainerManager.getForPath("Labs/Bimber");
+        UserSchema us2 = QueryService.get().getUserSchema(_containerUser.getUser(), sourceContainer, "bimber_data");
+        if (us2 == null)
+        {
+            throw new PipelineJobException("Missing bimber_data schema");
+        }
+
+        List<String> allIds = new ArrayList<>(new TableSelector(us2.getTable("subjects"), PageFlowUtil.set("Rh"), null, null).getArrayList(String.class));
+
+        allIds.removeAll(existingIds);
+
+        if (allIds.isEmpty())
+        {
+            return null;
+        }
+
+        pipelineJob.getLogger().info("Creating {} subjects", allIds.size());
+        List<Map<String, Object>> toInsert = new ArrayList<>();
+        allIds.forEach(id -> {
+            toInsert.add(Map.of("Id", id));
+        });
+
+        try
+        {
+            BatchValidationException bve = new BatchValidationException();
+            us.getTable("demographics").getUpdateService().insertRows(_containerUser.getUser(), _containerUser.getContainer(), toInsert, bve, null, null);
+            if (bve.hasErrors())
+            {
+                throw bve;
+            }
+        }
+        catch (BatchValidationException | SQLException | DuplicateKeyException | QueryUpdateServiceException e)
+        {
+            throw new PipelineJobException(e);
+        }
+
+        return null;
+    }
+
+    @Override
+    public List<String> getRequiredSettings()
+    {
+        return List.of();
+    }
+
+    @Override
+    public void setSettings(Map<String, String> map) throws XmlException
+    {
+
+    }
+
+    @Override
+    public void setContainerUser(ContainerUser containerUser)
+    {
+        _containerUser = containerUser;
+    }
+}

From 7889a368fb9e62abedae8d16755054287a99fbd5 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 12:45:11 -0700
Subject: [PATCH 09/40] Bugfix to sample SQL and augment ETL

---
 .../queries/bimber_data/idrSampleSource.sql        | 14 ++++++++++----
 SivStudies/resources/etls/idr-data.xml             |  2 +-
 .../resources/queries/study/samples.query.xml      | 12 ++++++++++++
 .../resources/queries/study/samples/.qview.xml     |  4 ++++
 .../study/datasets/datasets_metadata.xml           | 12 ++++++++++++
 5 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/IDR/resources/queries/bimber_data/idrSampleSource.sql b/IDR/resources/queries/bimber_data/idrSampleSource.sql
index e9a7baa68..bed1a30ae 100644
--- a/IDR/resources/queries/bimber_data/idrSampleSource.sql
+++ b/IDR/resources/queries/bimber_data/idrSampleSource.sql
@@ -5,7 +5,10 @@ ID as sampleid,
 SampleDate as date,
 Tissue as sampleType,
 CellCnt as quantity,
-
+freezer,
+rack,
+box,
+"position",
 'Hansen/IDR' as dataSource
 
 FROM bimber_data.ln_loc
@@ -20,12 +23,15 @@ ID as sampleid,
 SampleDate as date,
 CASE
     WHEN (Tissue IS NOT NULL AND Tissue != '') AND (SampleType IS NOT NULL AND SampleType != '') THEN (SampleType || ' / ' || Tissue)
-    WHEN (Tissue IS NOT NULL AND Tissue = '') THEN Tissue
-    WHEN (SampleType IS NOT NULL AND Tissue = '') THEN SampleType
+    WHEN (Tissue IS NOT NULL AND Tissue != '') THEN Tissue
+    WHEN (SampleType IS NOT NULL AND SampleType != '') THEN SampleType
     ELSE NULL
 END AS sampleType,
 null as quantity,
-
+freezer,
+rack,
+box,
+"position",
 'Hansen/IDR' as dataSource
 
 FROM bimber_data.ult_loc
diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml
index aa9aaf086..066afee50 100644
--- a/SivStudies/resources/etls/idr-data.xml
+++ b/SivStudies/resources/etls/idr-data.xml
@@ -37,7 +37,7 @@
                     <setting name="dataSourceSchema" value="bimber_data"/>
                     <setting name="dataSourceQuery" value="idrSampleSource"/>
                     <setting name="dataSourceSubjectColumn" value="Id"/>
-                    <setting name="dataSourceColumns" value="Id,date,sampleId,sampleType,quantity"/>
+                    <setting name="dataSourceColumns" value="Id,date,sampleId,sampleType,quantity,freezer,rack,box,position"/>
                     <setting name="dataSourceColumnDefaults" value="dataSource=Hansen/IDR"/>
 
                     <setting name="targetSchema" value="study"/>
diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml
index 3aaacd32e..3d5fc433c 100644
--- a/SivStudies/resources/queries/study/samples.query.xml
+++ b/SivStudies/resources/queries/study/samples.query.xml
@@ -23,6 +23,18 @@
                     <column columnName="quantity_units">
                         <columnTitle>Quantity Units</columnTitle>
                     </column>
+                    <column columnName="freezer">
+                        <columnTitle>Freezer</columnTitle>
+                    </column>
+                    <column columnName="rack">
+                        <columnTitle>Rack</columnTitle>
+                    </column>
+                    <column columnName="box">
+                        <columnTitle>Box</columnTitle>
+                    </column>
+                    <column columnName="position">
+                        <columnTitle>Position</columnTitle>
+                    </column>
                     <column columnName="comments">
                         <columnTitle>Comments</columnTitle>
                         <inputType>textarea</inputType>
diff --git a/SivStudies/resources/queries/study/samples/.qview.xml b/SivStudies/resources/queries/study/samples/.qview.xml
index 0e70ce34d..31300da7c 100644
--- a/SivStudies/resources/queries/study/samples/.qview.xml
+++ b/SivStudies/resources/queries/study/samples/.qview.xml
@@ -7,6 +7,10 @@
         <column name="quantity"/>
         <column name="quantity_units"/>
         <column name="sampleId"/>
+        <column name="freezer"/>
+        <column name="rack"/>
+        <column name="box"/>
+        <column name="position"/>
         <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 181994629..f6adf7cf0 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -372,6 +372,18 @@
             <column columnName="quantity_units">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="freezer">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="rack">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="box">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="position">
+                <datatype>varchar</datatype>
+            </column>
             <column columnName="comments">
                 <datatype>varchar</datatype>
             </column>

From 9d41688487b47695d847680c56d66da098df27b0 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 12:46:56 -0700
Subject: [PATCH 10/40] Add enddate to pvl_outcomes

---
 SivStudies/resources/queries/study/pvl_outcomes.query.xml   | 6 +++++-
 SivStudies/resources/queries/study/pvl_outcomes/.qview.xml  | 1 +
 .../referenceStudy/study/datasets/datasets_metadata.xml     | 3 +++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/SivStudies/resources/queries/study/pvl_outcomes.query.xml b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
index e06038802..c61fefb74 100644
--- a/SivStudies/resources/queries/study/pvl_outcomes.query.xml
+++ b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
@@ -5,7 +5,11 @@
                 <columns>
                     <column columnName="Id"/>
                     <column columnName="date">
-                        <columnTitle>Date</columnTitle>
+                        <columnTitle>Window Start</columnTitle>
+                        <formatString>Date</formatString>
+                    </column>
+                    <column columnName="enddate">
+                        <columnTitle>Window End</columnTitle>
                         <formatString>Date</formatString>
                     </column>
                     <column columnName="outcome">
diff --git a/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml b/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml
index 05ad5dcfa..8be5fbb72 100644
--- a/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml
+++ b/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml
@@ -2,6 +2,7 @@
     <columns>
         <column name="Id"/>
         <column name="date"/>
+        <column name="enddate"/>
         <column name="outcome"/>
         <column name="numeric_value"/>
         <column name="string_value"/>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index f6adf7cf0..84867e793 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -563,6 +563,9 @@
                 <datatype>timestamp</datatype>
                 <conceptURI>http://cpas.labkey.com/laboratory#sampleDate</conceptURI>
             </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+            </column>
             <column columnName="dataSource">
                 <datatype>varchar</datatype>
             </column>

From 372b562912aab3eed96232e49c27e90d86880ecc Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 13:36:12 -0700
Subject: [PATCH 11/40] Remove duplicate IDs in AddMissingIdrSubjects

---
 .../src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
index c0873e312..2aaa4407f 100644
--- a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
+++ b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
@@ -2,6 +2,7 @@
 
 import org.apache.xmlbeans.XmlException;
 import org.jetbrains.annotations.NotNull;
+import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
 import org.labkey.api.data.TableSelector;
@@ -49,12 +50,13 @@ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJo
         List<String> allIds = new ArrayList<>(new TableSelector(us2.getTable("subjects"), PageFlowUtil.set("Rh"), null, null).getArrayList(String.class));
 
         allIds.removeAll(existingIds);
-
         if (allIds.isEmpty())
         {
             return null;
         }
 
+        allIds = new ArrayList<>(new CaseInsensitiveHashSet(allIds));
+
         pipelineJob.getLogger().info("Creating {} subjects", allIds.size());
         List<Map<String, Object>> toInsert = new ArrayList<>();
         allIds.forEach(id -> {

From d2befb6908d7a95f3586620e7c0197610850e4ee Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 13:36:52 -0700
Subject: [PATCH 12/40] Bugfix to AddMissingIdrSubjects

---
 .../src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
index 2aaa4407f..fb8f9de4c 100644
--- a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
+++ b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
@@ -52,7 +52,7 @@ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJo
         allIds.removeAll(existingIds);
         if (allIds.isEmpty())
         {
-            return null;
+            return new RecordedActionSet();
         }
 
         allIds = new ArrayList<>(new CaseInsensitiveHashSet(allIds));
@@ -77,7 +77,7 @@ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJo
             throw new PipelineJobException(e);
         }
 
-        return null;
+        return new RecordedActionSet();
     }
 
     @Override

From 7dc523c92df4f4e74a35a0d0764d2dc5b739e48f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 10:01:05 -0700
Subject: [PATCH 13/40] Expand study triggers and update cohort fields

---
 SivStudies/resources/etls/idr-data.xml        |   2 +-
 .../queries/study/assignment.query.xml        |  13 +-
 .../queries/study/assignment/.qview.xml       |   3 +-
 .../study/datasets/datasets_metadata.xml      |   5 +-
 .../query/DefaultDatasetTrigger.java          |  87 +++++++++++++
 .../query/NumericValuesTrigger.java           | 119 ++++++++++++++++++
 .../query/SivStudiesCustomizer.java           |  21 ++++
 7 files changed, 245 insertions(+), 5 deletions(-)
 create mode 100644 SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
 create mode 100644 SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java

diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml
index 066afee50..909ee84ec 100644
--- a/SivStudies/resources/etls/idr-data.xml
+++ b/SivStudies/resources/etls/idr-data.xml
@@ -15,7 +15,7 @@
                     <setting name="dataSourceQuery" value="subjects"/>
                     <setting name="dataSourceSubjectColumn" value="Rh"/>
                     <setting name="dataSourceColumns" value="Rh,cohortStart,cohortEnd,cohort,RhCode"/>
-                    <setting name="dataSourceColumnMapping" value="Rh=Id,cohortStart=date,cohortEnd=enddate,RhCode=cohortId,cohort=study"/>
+                    <setting name="dataSourceColumnMapping" value="Rh=Id,cohortStart=date,cohortEnd=enddate,RhCode=cohortAlias,cohort=study"/>
                     <setting name="dataSourceColumnDefaults" value="dataSource=Hansen/IDR"/>
                     <setting name="dataSourceAdditionalFilters" value="cohortStart~neq=0000-00-00"/>
 
diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml
index ed31de818..85ef69650 100644
--- a/SivStudies/resources/queries/study/assignment.query.xml
+++ b/SivStudies/resources/queries/study/assignment.query.xml
@@ -18,8 +18,8 @@
                     <column columnName="subgroup">
                         <columnTitle>Sub-Group</columnTitle>
                     </column>
-                    <column columnName="cohortId">
-                        <columnTitle>Cohort ID</columnTitle>
+                    <column columnName="cohortAlias">
+                        <columnTitle>Cohort Alias</columnTitle>
                     </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
@@ -27,6 +27,15 @@
                     <column columnName="category">
                         <columnTitle>Category</columnTitle>
                     </column>
+                    <column columnName="cohortId">
+                        <columnTitle>Cohort ID</columnTitle>
+                        <fk>
+                            <fkDbSchema>studies</fkDbSchema>
+                            <fkTable>studyCohorts</fkTable>
+                            <fkColumnName>rowId</fkColumnName>
+                            <fkDisplayColumnName>label</fkDisplayColumnName>
+                        </fk>
+                    </column>
                     <column columnName="description">
                         <columnTitle>Description</columnTitle>
                         <isHidden>true</isHidden>
diff --git a/SivStudies/resources/queries/study/assignment/.qview.xml b/SivStudies/resources/queries/study/assignment/.qview.xml
index b10c793d3..d060c788c 100644
--- a/SivStudies/resources/queries/study/assignment/.qview.xml
+++ b/SivStudies/resources/queries/study/assignment/.qview.xml
@@ -5,8 +5,9 @@
         <column name="enddate"/>
         <column name="study"/>
         <column name="subgroup"/>
-        <column name="cohortId"/>
+        <column name="cohortAlias"/>
         <column name="category"/>
+        <column name="cohortId"/>
         <column name="dataSource"/>
     </columns>
     <sorts>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 84867e793..2e2124fe7 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -330,9 +330,12 @@
             <column columnName="subgroup">
                 <datatype>varchar</datatype>
             </column>
-            <column columnName="cohortId">
+            <column columnName="cohortAlias">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="cohortId">
+                <datatype>integer</datatype>
+            </column>
             <column columnName="description">
                 <datatype>varchar</datatype>
             </column>
diff --git a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
new file mode 100644
index 000000000..3d5b21a0b
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
@@ -0,0 +1,87 @@
+package org.labkey.sivstudies.query;
+
+import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.triggers.Trigger;
+import org.labkey.api.data.triggers.TriggerFactory;
+import org.labkey.api.query.ValidationException;
+import org.labkey.api.security.User;
+import org.labkey.api.util.logging.LogHelper;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class DefaultDatasetTrigger implements Trigger
+{
+    protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to DefaultDatasetTrigger");
+
+    public static class Factory implements TriggerFactory
+    {
+        public Factory()
+        {
+
+        }
+
+        @Override
+        public @NotNull Collection<Trigger> createTrigger(@Nullable Container c, TableInfo table, Map<String, Object> extraContext)
+        {
+            return List.of(new DefaultDatasetTrigger());
+        }
+
+    }
+
+    @Override
+    public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        beforeInsert(table, c, user, newRow, errors, extraContext, null);
+    }
+
+    @Override
+    public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext, @Nullable Map<String, Object> existingRecord) throws ValidationException
+    {
+        beforeUpsert(table, c, user, newRow, existingRecord, errors, extraContext);
+    }
+
+    @Override
+    public void beforeUpdate(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        beforeUpsert(table, c, user, newRow, oldRow, errors, extraContext);
+    }
+
+    private void beforeUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        if (newRow == null)
+        {
+            _log.error("newRow was null. Unsure when this would ever happen", new Exception());
+            return;
+        }
+
+        // Simplify properties:
+        mergeOldToNewRow(newRow, oldRow);
+
+        doBeforeUpsert(table, c, user, newRow, oldRow, errors, extraContext);
+    }
+
+    protected void doBeforeUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        // Allow subclasses to implement code here
+    }
+
+    private void mergeOldToNewRow(@NotNull Map<String, Object> newRow, @Nullable Map<String, Object> oldRow) throws ValidationException
+    {
+        if (oldRow != null)
+        {
+            for (String propName : oldRow.keySet())
+            {
+                if (!newRow.containsKey(propName))
+                {
+                    newRow.put(propName, oldRow.get(propName));
+                }
+            }
+        }
+    }
+}
diff --git a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
new file mode 100644
index 000000000..83f442613
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
@@ -0,0 +1,119 @@
+package org.labkey.sivstudies.query;
+
+import org.apache.commons.lang3.math.NumberUtils;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.ColumnInfo;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.triggers.Trigger;
+import org.labkey.api.data.triggers.TriggerFactory;
+import org.labkey.api.query.SimpleValidationError;
+import org.labkey.api.query.ValidationException;
+import org.labkey.api.security.User;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class NumericValuesTrigger extends DefaultDatasetTrigger
+{
+    public static class Factory implements TriggerFactory
+    {
+        // This map allows caller to supply a list of <StringValue> -> <TargetField>. If that string is found in a numeric field, it will be
+        private final List<StringTransformer> _stringTransformers;
+
+        public Factory()
+        {
+            this(null);
+        }
+
+        public Factory(@Nullable List<StringTransformer> stringTransformers)
+        {
+            _stringTransformers = stringTransformers == null ? Collections.emptyList() : stringTransformers;
+        }
+
+        @Override
+        public @NotNull Collection<Trigger> createTrigger(@Nullable Container c, TableInfo table, Map<String, Object> extraContext)
+        {
+            return List.of(new NumericValuesTrigger(_stringTransformers));
+        }
+    }
+
+    private final List<StringTransformer> _stringTransformers;
+
+    public NumericValuesTrigger(List<StringTransformer> stringTransformers)
+    {
+        _stringTransformers = stringTransformers;
+    }
+
+    @Override
+    protected void doBeforeUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        inspectNumericValues(table, newRow, errors);
+    }
+
+    private void inspectNumericValues(TableInfo table, Map<String, Object> row, ValidationException errors)
+    {
+        for (String propName : row.keySet())
+        {
+            if (row.get(propName) == null)
+            {
+                continue;
+            }
+
+            ColumnInfo ci = table.getColumn(propName);
+            if (ci == null)
+            {
+                continue;
+            }
+
+            if (!Number.class.isAssignableFrom(ci.getJdbcType().getJavaClass()))
+            {
+                continue;
+            }
+
+            String val = String.valueOf(row.get(propName));
+            if (NumberUtils.isCreatable(val))
+            {
+                return;
+            }
+
+            // commas are a common problem:
+            val = val.replaceAll(",", "");
+            if (NumberUtils.isCreatable(val))
+            {
+                row.put(propName, val);
+                return;
+            }
+
+            // The Rlabkey API sending NAs as strings is another common problem:
+            if ("NA".equalsIgnoreCase(val))
+            {
+                row.put(propName, null);
+                return;
+            }
+
+            for (StringTransformer stringTransformer : _stringTransformers)
+            {
+                stringTransformer.inspectValue(table, row, val, propName, errors);
+                val = String.valueOf(row.get(propName));
+            }
+
+            if (NumberUtils.isCreatable(val))
+            {
+                row.put(propName, val);
+                return;
+            }
+
+            errors.addError(new SimpleValidationError("Non-numeric value for field " + propName + ": " + val, propName, ValidationException.SEVERITY.ERROR));
+        }
+    }
+
+    public interface StringTransformer
+    {
+        // This method allows code to inspect and modify non-numeric values
+        public void inspectValue(TableInfo ti, Map<String, Object> row, String stringValue, String propName, ValidationException errors);
+    }
+}
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 9d82243ac..6e59d69a1 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -1,5 +1,6 @@
 package org.labkey.sivstudies.query;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.data.AbstractTableInfo;
@@ -69,6 +70,7 @@ public void performDatasetCustomization(DatasetTable ds)
 
             appendDemographicsColumns(ati);
 
+            addNumericValuesTrigger(ati);
             if ("viralLoads".equalsIgnoreCase(ds.getName()))
             {
                 customizeViralLoads(ati);
@@ -494,6 +496,25 @@ private BaseColumnInfo getWrappedIdCol(UserSchema targetQueryUserSchema, String
         return col;
     }
 
+    private void addNumericValuesTrigger(AbstractTableInfo ati)
+    {
+        List<NumericValuesTrigger.StringTransformer> stringTransformers = new ArrayList<>();
+        if ("immunizations".equalsIgnoreCase(ati.getName()))
+        {
+            stringTransformers.add((ti, row, stringValue, propName, errors) -> {
+                if ("quantity".equalsIgnoreCase(propName) & "Supernatant".equalsIgnoreCase(stringValue))
+                {
+                    row.put("quantity", null);
+                    String comments = row.get("comments") == null ? null : StringUtils.trimToNull(String.valueOf(row.get("comments")));
+                    comments = (comments == null ? "" : comments + ", ") + "Quantity: Supernatant";
+                    row.put("comments", comments);
+                }
+            });
+        }
+
+        ati.addTriggerFactory(new NumericValuesTrigger.Factory(stringTransformers));
+    }
+
     private void customizeViralLoads(AbstractTableInfo ati)
     {
         ati.addTriggerFactory(new ViralLoadsTriggerFactory());

From 5c54e539f7bf2157f11fcd27e3b5c8b0b9328a92 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 10:26:46 -0700
Subject: [PATCH 14/40] Bugfix to ART fields

---
 SivStudies/resources/views/dataNotes.html                 | 6 ++++++
 SivStudies/resources/views/studiesAdmin.html              | 3 +++
 .../org/labkey/sivstudies/query/SivStudiesCustomizer.java | 8 ++++----
 3 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 SivStudies/resources/views/dataNotes.html

diff --git a/SivStudies/resources/views/dataNotes.html b/SivStudies/resources/views/dataNotes.html
new file mode 100644
index 000000000..0f56d3005
--- /dev/null
+++ b/SivStudies/resources/views/dataNotes.html
@@ -0,0 +1,6 @@
+This page summarizes
+<ul>
+    <li>The fields to calculate timePostSivChallenge require a record in studies.subjectAnchorDates from the same Id where eventLabel = 'SIV Infection'. There may be records in the treatments table for SIV challenges; these do not count</li>
+    <li>The fields to calculate overlapping PVLs require a record in the viral_load table from the same Id/date, where target = SIV and where sampleType = plasma.</li>
+    <li>The fields to calculate ART-related date require record(s) in the treatments table from the same Id, where category = 'ART'</li>
+</ul>
\ No newline at end of file
diff --git a/SivStudies/resources/views/studiesAdmin.html b/SivStudies/resources/views/studiesAdmin.html
index fb5e91025..8cebaf45b 100644
--- a/SivStudies/resources/views/studiesAdmin.html
+++ b/SivStudies/resources/views/studiesAdmin.html
@@ -27,6 +27,9 @@
                 },{
                     name: 'Notification Admin',
                     url: LABKEY.ActionURL.buildURL('ldk', 'notificationAdmin.view')
+                },{
+                    name: 'Notes For Managing Data',
+                    url: LABKEY.ActionURL.buildURL('sivstudies', 'dataNotes.view')
                 }]
             }]
         }]
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 6e59d69a1..63293efa4 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -427,13 +427,13 @@ public TableInfo getLookupTableInfo()
                 UserSchema targetSchema = ds.getUserSchema().getDefaultSchema().getUserSchema(targetSchemaName);
                 QueryDefinition qd = QueryService.get().createQueryDef(u, targetSchemaContainer, targetSchema, name);
                 qd.setSql("SELECT\n" +
-                        "max(tr.date) as artInitiation,\n" +
-                        "CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(tr.date) AS DATE), CAST(c." + dateColName + " AS DATE)), INTEGER) as daysPostArtInitiation,\n" +
-                        "CONVERT(age_in_months(CAST(max(tr.date) AS DATE), CAST(c." + dateColName + " AS DATE)), FLOAT) as monthsPostArtInitiation,\n" +
+                        "min(tr.date) as artInitiation,\n" +
+                        "CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(min(tr.date) AS DATE), CAST(c." + dateColName + " AS DATE)), INTEGER) as daysPostArtInitiation,\n" +
+                        "CONVERT(age_in_months(CAST(min(tr.date) AS DATE), CAST(c." + dateColName + " AS DATE)), FLOAT) as monthsPostArtInitiation,\n" +
                         "max(tr.enddate) as artRelease,\n" +
                         "CONVERT(CASE WHEN max(tr.enddate) IS NULL THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(tr.enddate) AS DATE), CAST(c." + dateColName + " AS DATE)) END, INTEGER) as daysPostArtRelease,\n" +
                         "CONVERT(CASE WHEN max(tr.enddate) IS NULL THEN NULL ELSE age_in_months(CAST(max(tr.enddate) AS DATE), CAST(c." + dateColName + " AS DATE)) END, FLOAT) as monthsPostArtRelease,\n" +
-                        "CAST(CASE WHEN CAST(max(tr.date) AS DATE) < CAST(c." + dateCol.getFieldKey().toString()  + " AS DATE) AND CAST(max(coalesce(tr.enddate, now())) AS DATE) >= CAST(c." + dateCol.getFieldKey().toString() + " AS DATE) THEN 'Y' ELSE null END as VARCHAR) as onArt,\n" +
+                        "CAST(CASE WHEN CAST(min(tr.date) AS DATE) <= CAST(c." + dateCol.getFieldKey().toString()  + " AS DATE) AND CAST(max(coalesce(tr.enddate, now())) AS DATE) >= CAST(c." + dateCol.getFieldKey().toString() + " AS DATE) THEN 'Y' ELSE null END as VARCHAR) as onArt,\n" +
                         "GROUP_CONCAT(DISTINCT tr.treatment) AS artTreatment,\n" +
                         "c." + pkCol.getFieldKey().toString() + "\n" +
                         "FROM \"" + schemaName + "\".\"" + queryName + "\" c " +

From 8c1710710e4956990bffe0ea7b28ead192889362 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 11:03:35 -0700
Subject: [PATCH 15/40] Bugfix to NumericValuesTrigger

---
 .../org/labkey/sivstudies/query/NumericValuesTrigger.java    | 2 +-
 .../org/labkey/sivstudies/query/SivStudiesCustomizer.java    | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
index 83f442613..376bd7a23 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
@@ -101,7 +101,7 @@ private void inspectNumericValues(TableInfo table, Map<String, Object> row, Vali
                 val = String.valueOf(row.get(propName));
             }
 
-            if (NumberUtils.isCreatable(val))
+            if (val == null || NumberUtils.isCreatable(val))
             {
                 row.put(propName, val);
                 return;
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 63293efa4..03d75117b 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -75,6 +75,11 @@ public void performDatasetCustomization(DatasetTable ds)
             {
                 customizeViralLoads(ati);
             }
+
+            if ("assignment".equalsIgnoreCase(ds.getName()))
+            {
+                ati.addTriggerFactory(StudiesService.get().getStudiesTriggerFactory());
+            }
         }
         else
         {

From 3d826242993c0f49db252ec481574462c3117f0e Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 14:05:46 -0700
Subject: [PATCH 16/40] Clean up trigger/customizer layer code

---
 SivStudies/resources/queries/study/studyData.query.xml          | 1 +
 .../src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml
index e78b0c84f..ab97c463c 100644
--- a/SivStudies/resources/queries/study/studyData.query.xml
+++ b/SivStudies/resources/queries/study/studyData.query.xml
@@ -7,6 +7,7 @@
                     <column columnName="Id">
                         <conceptURI>http://cpas.labkey.com/Study#ParticipantId</conceptURI>
                         <url>laboratory/dataBrowser.view?subjectId=${Id}</url>
+                        <facetingBehavior>ALWAYS_OFF</facetingBehavior>
                     </column>
                     <column columnName="date">
                         <columnTitle>Date</columnTitle>
diff --git a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
index 3d5b21a0b..26efb1912 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
@@ -77,7 +77,7 @@ private void mergeOldToNewRow(@NotNull Map<String, Object> newRow, @Nullable Map
         {
             for (String propName : oldRow.keySet())
             {
-                if (!newRow.containsKey(propName))
+                if (!newRow.containsKey(propName) & oldRow.get(propName) != null)
                 {
                     newRow.put(propName, oldRow.get(propName));
                 }

From 3239871378ac285ddc2ffb0adea85bcd8dd51284 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 14:29:06 -0700
Subject: [PATCH 17/40] Clean up trigger/customizer layer code

---
 .../org/labkey/sivstudies/query/NumericValuesTrigger.java   | 6 +++---
 .../org/labkey/sivstudies/query/SivStudiesCustomizer.java   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
index 376bd7a23..38eac57b8 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
@@ -74,7 +74,7 @@ private void inspectNumericValues(TableInfo table, Map<String, Object> row, Vali
                 continue;
             }
 
-            String val = String.valueOf(row.get(propName));
+            String val = row.get(propName) == null ? null : String.valueOf(row.get(propName));
             if (NumberUtils.isCreatable(val))
             {
                 return;
@@ -89,7 +89,7 @@ private void inspectNumericValues(TableInfo table, Map<String, Object> row, Vali
             }
 
             // The Rlabkey API sending NAs as strings is another common problem:
-            if ("NA".equalsIgnoreCase(val))
+            if ("NA".equalsIgnoreCase(val) || "null".equalsIgnoreCase(val))
             {
                 row.put(propName, null);
                 return;
@@ -98,7 +98,7 @@ private void inspectNumericValues(TableInfo table, Map<String, Object> row, Vali
             for (StringTransformer stringTransformer : _stringTransformers)
             {
                 stringTransformer.inspectValue(table, row, val, propName, errors);
-                val = String.valueOf(row.get(propName));
+                val = row.get(propName) == null ? null : String.valueOf(row.get(propName));
             }
 
             if (val == null || NumberUtils.isCreatable(val))
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 03d75117b..69a0174e9 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -507,7 +507,7 @@ private void addNumericValuesTrigger(AbstractTableInfo ati)
         if ("immunizations".equalsIgnoreCase(ati.getName()))
         {
             stringTransformers.add((ti, row, stringValue, propName, errors) -> {
-                if ("quantity".equalsIgnoreCase(propName) & "Supernatant".equalsIgnoreCase(stringValue))
+                if ("quantity".equalsIgnoreCase(propName) & ("Supernatant".equalsIgnoreCase(stringValue) | "Supernatent".equalsIgnoreCase(stringValue)))
                 {
                     row.put("quantity", null);
                     String comments = row.get("comments") == null ? null : StringUtils.trimToNull(String.valueOf(row.get("comments")));

From 424c665b88d546bec2bc48086767d15896b2d6da Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 14:46:36 -0700
Subject: [PATCH 18/40] Create fields to coalesce name/label for studies

---
 SivStudies/resources/queries/study/assignment.query.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml
index 85ef69650..00df5943a 100644
--- a/SivStudies/resources/queries/study/assignment.query.xml
+++ b/SivStudies/resources/queries/study/assignment.query.xml
@@ -33,7 +33,7 @@
                             <fkDbSchema>studies</fkDbSchema>
                             <fkTable>studyCohorts</fkTable>
                             <fkColumnName>rowId</fkColumnName>
-                            <fkDisplayColumnName>label</fkDisplayColumnName>
+                            <fkDisplayColumnName>labelOrName</fkDisplayColumnName>
                         </fk>
                     </column>
                     <column columnName="description">

From db5d771ecc846bfa03fd6228df87c99d625b1184 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 15:22:33 -0700
Subject: [PATCH 19/40] Expand SIV studies notification

---
 .../study/pvlWithoutInfectionDate.query.xml   |  9 ++++++
 .../queries/study/pvlWithoutInfectionDate.sql | 10 +++++++
 .../SivStudiesDataValidationNotification.java | 29 +++++++++----------
 3 files changed, 32 insertions(+), 16 deletions(-)
 create mode 100644 SivStudies/resources/queries/study/pvlWithoutInfectionDate.query.xml
 create mode 100644 SivStudies/resources/queries/study/pvlWithoutInfectionDate.sql

diff --git a/SivStudies/resources/queries/study/pvlWithoutInfectionDate.query.xml b/SivStudies/resources/queries/study/pvlWithoutInfectionDate.query.xml
new file mode 100644
index 000000000..bd636239a
--- /dev/null
+++ b/SivStudies/resources/queries/study/pvlWithoutInfectionDate.query.xml
@@ -0,0 +1,9 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="pvlWithoutInfectionDate" tableDbType="NOT_IN_DB">
+                <tableTitle>Animals With PVL Data But No Infection Date</tableTitle>
+            </table>
+        </tables>
+    </metadata>
+</query>
diff --git a/SivStudies/resources/queries/study/pvlWithoutInfectionDate.sql b/SivStudies/resources/queries/study/pvlWithoutInfectionDate.sql
new file mode 100644
index 000000000..1a655cbd8
--- /dev/null
+++ b/SivStudies/resources/queries/study/pvlWithoutInfectionDate.sql
@@ -0,0 +1,10 @@
+SELECT
+    vl.Id,
+    max(vl.result) as maxViralLoad
+
+FROM study.viralloads vl
+WHERE
+    vl.result IS NOT NULL AND
+    ((vl.lod is not NULL AND vl.result > vl.lod) OR (vl.lod IS NULL AND vl.result > 50)) AND
+    vl.timePostSivChallenge.infectionDate IS NULL
+GROUP BY vl.Id
\ No newline at end of file
diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
index 314c662a0..faeae93c4 100644
--- a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
+++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
@@ -63,6 +63,8 @@ public String getEmailSubject(Container c)
         Date now = new Date();
 
         duplicateInfectionCheck(c, u, msg);
+        infectionAnchorDateDiscordance(c, u, msg);
+        pvlWithoutInfectionDate(c, u, msg);
 
         if (!msg.isEmpty())
         {
@@ -91,35 +93,30 @@ private TableInfo getTableInfo(User u, Container c, String schemaName, String qu
 
     private void duplicateInfectionCheck(Container c, User u, StringBuilder msg)
     {
-        String schemaName = "study";
-        String queryName = "duplicateInfectionDates";
-
-        TableInfo ti = getTableInfo(u, c, schemaName, queryName);
-
-        TableSelector ts = new TableSelector(ti);
-        long count = ts.getRowCount();
-        if (count > 0)
-        {
-            msg.append("<b>WARNING: There are ").append(count).append(" duplicate infection date records</b><br>\n");
-            msg.append("<p><a href='").append(getExecuteQueryUrl(c, schemaName, queryName, null)).append("'>Click here to view them</a><br>\n\n");
-            msg.append("<hr>\n\n");
-        }
+        genericQueryCheck(c, u, msg, "study", "duplicateInfectionDates", "duplicate infection date records");
     }
 
     private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder msg)
     {
-        String schemaName = "study";
-        String queryName = "infectionAnchorDateDiscordance";
+        genericQueryCheck(c, u, msg, "study", "infectionAnchorDateDiscordance", "records with discordant treatment and anchor date SIV infection records");
+    }
 
+    private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message)
+    {
         TableInfo ti = getTableInfo(u, c, schemaName, queryName);
 
         TableSelector ts = new TableSelector(ti);
         long count = ts.getRowCount();
         if (count > 0)
         {
-            msg.append("<b>WARNING: There are ").append(count).append(" records with discordant treatment and anchor date SIV infection records</b><br>\n");
+            msg.append("<b>WARNING: There are ").append(count).append(" " + message + "</b><br>\n");
             msg.append("<p><a href='").append(getExecuteQueryUrl(c, schemaName, queryName, null)).append("'>Click here to view them</a><br>\n\n");
             msg.append("<hr>\n\n");
         }
     }
+
+    private void pvlWithoutInfectionDate(Container c, User u, StringBuilder msg)
+    {
+        genericQueryCheck(c, u, msg, "study", "pvlWithoutInfectionDate", "animals with PVL data but no record of SIV infection");
+    }
 }

From 6d8c507311c98070bfa1c3941b68effbb20a7533 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 20 Sep 2025 07:29:09 -0700
Subject: [PATCH 20/40] Dont add NumericValuesTrigger to PVL table

---
 .../org/labkey/sivstudies/query/SivStudiesCustomizer.java   | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 69a0174e9..4a8f3e36d 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -503,6 +503,12 @@ private BaseColumnInfo getWrappedIdCol(UserSchema targetQueryUserSchema, String
 
     private void addNumericValuesTrigger(AbstractTableInfo ati)
     {
+        // This behavior conflicts with ViralLoadsTriggerFactory
+        if ("viralLoads".equalsIgnoreCase(ati.getName()))
+        {
+            return;
+        }
+
         List<NumericValuesTrigger.StringTransformer> stringTransformers = new ArrayList<>();
         if ("immunizations".equalsIgnoreCase(ati.getName()))
         {

From bfabbd2a82f4f8088413501162b71929de761053 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 27 Sep 2025 09:06:11 -0700
Subject: [PATCH 21/40] Add QCLabel filter

---
 PMR/resources/etls/prime-chemistryResults.xml   | 3 +++
 PMR/resources/etls/prime-clinpathRuns.xml       | 3 +++
 PMR/resources/etls/prime-hematologyResults.xml  | 3 +++
 PMR/resources/etls/prime-histology.xml          | 3 +++
 PMR/resources/etls/prime-microbiology.xml       | 3 +++
 PMR/resources/etls/prime-pathologyDiagnoses.xml | 3 +++
 6 files changed, 18 insertions(+)

diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
index 0c0cd1c92..dc8666de7 100644
--- a/PMR/resources/etls/prime-chemistryResults.xml
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -18,6 +18,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="chemistryResults" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-clinpathRuns.xml b/PMR/resources/etls/prime-clinpathRuns.xml
index f6afe0b27..42af3a095 100644
--- a/PMR/resources/etls/prime-clinpathRuns.xml
+++ b/PMR/resources/etls/prime-clinpathRuns.xml
@@ -19,6 +19,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="clinpathRuns" bulkLoad="true" targetOption="merge" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
index 9df7326e3..510f69be4 100644
--- a/PMR/resources/etls/prime-hematologyResults.xml
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -18,6 +18,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="hematologyResults" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-histology.xml b/PMR/resources/etls/prime-histology.xml
index 27a21350b..9f59b0b3b 100644
--- a/PMR/resources/etls/prime-histology.xml
+++ b/PMR/resources/etls/prime-histology.xml
@@ -17,6 +17,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="histology" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-microbiology.xml b/PMR/resources/etls/prime-microbiology.xml
index 194c4d83e..d49d51582 100644
--- a/PMR/resources/etls/prime-microbiology.xml
+++ b/PMR/resources/etls/prime-microbiology.xml
@@ -18,6 +18,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="microbiology" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-pathologyDiagnoses.xml b/PMR/resources/etls/prime-pathologyDiagnoses.xml
index 428ef6e3a..172297dcb 100644
--- a/PMR/resources/etls/prime-pathologyDiagnoses.xml
+++ b/PMR/resources/etls/prime-pathologyDiagnoses.xml
@@ -16,6 +16,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="pathologyDiagnoses" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>

From 8564e48a32af576f039f20a1f49400485fd60450 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 29 Sep 2025 11:19:36 -0700
Subject: [PATCH 22/40] Add field for number of PVL datapoints

---
 SivStudies/resources/queries/study/pvl_outcomes.query.xml     | 4 ++++
 .../referenceStudy/study/datasets/datasets_metadata.xml       | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/SivStudies/resources/queries/study/pvl_outcomes.query.xml b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
index c61fefb74..53f67bfff 100644
--- a/SivStudies/resources/queries/study/pvl_outcomes.query.xml
+++ b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
@@ -21,6 +21,10 @@
                     <column columnName="string_value">
                         <columnTitle>String Value</columnTitle>
                     </column>
+                    <column columnName="numDatapoints">
+                        <columnTitle># Datapoints</columnTitle>
+                        <description>The number of PVL datapoints used for this calculation</description>
+                    </column>
                     <column columnName="comments">
                         <columnTitle>Comments</columnTitle>
                     </column>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 2e2124fe7..802e195fc 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -585,6 +585,9 @@
             <column columnName="string_value">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="numDatapoints">
+                <datatype>integer</datatype>
+            </column>
             <column columnName="comments">
                 <datatype>varchar</datatype>
             </column>

From 753cd1e6a0c94d7eb9168e8ae81144c4c2439cb3 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 29 Sep 2025 20:25:08 -0700
Subject: [PATCH 23/40] Increase batch size

---
 .../src/org/labkey/sivstudies/etl/SubjectScopedSelect.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
index 6c750b8bf..37b54fa8d 100644
--- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
+++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
@@ -101,7 +101,7 @@ public boolean isRequired()
         }
     }
 
-    final int BATCH_SIZE = 250;
+    final int BATCH_SIZE = 500;
 
     private MODE getMode()
     {

From 5c8c616ed79e80b4d06e558a1c8bd56814cc6deb Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 4 Oct 2025 07:44:12 -0700
Subject: [PATCH 24/40] Build short delay into github triggers to aid
 cross-repo commits

---
 .github/workflows/build.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fff5cbff1..535315bc0 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -10,6 +10,11 @@ jobs:
     if: github.repository == 'BimberLabInternal/BimberLabKeyModules'
     runs-on: ubuntu-latest
     steps:
+      # Note: use slight delay in case there are associated commits across repos
+      - name: "Sleep for 30 seconds"
+        run: sleep 30s
+        shell: bash
+
       - name: "Build DISCVR"
         uses: bimberlabinternal/DevOps/githubActions/discvr-build@master
         with:

From 23761ec7445f98b8c6db19e69ab9e99534b25ee1 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 6 Oct 2025 08:03:12 -0700
Subject: [PATCH 25/40] Switch ETLs to log row count discrepancies

---
 PMR/resources/etls/prime-blooddraws.xml         |  2 ++
 PMR/resources/etls/prime-chemistryResults.xml   |  2 ++
 PMR/resources/etls/prime-clinpathRuns.xml       |  2 ++
 PMR/resources/etls/prime-hematologyResults.xml  |  2 ++
 PMR/resources/etls/prime-histology.xml          |  2 ++
 PMR/resources/etls/prime-microbiology.xml       |  2 ++
 PMR/resources/etls/prime-pathologyDiagnoses.xml |  2 ++
 PMR/resources/etls/prime-weight.xml             |  2 ++
 .../org/labkey/primeseq/etl/VerifyRowCount.java | 17 +++++++++++++++--
 9 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/PMR/resources/etls/prime-blooddraws.xml b/PMR/resources/etls/prime-blooddraws.xml
index 13abeaa15..93d4a6cdf 100644
--- a/PMR/resources/etls/prime-blooddraws.xml
+++ b/PMR/resources/etls/prime-blooddraws.xml
@@ -34,6 +34,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="blood"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
index dc8666de7..380c88796 100644
--- a/PMR/resources/etls/prime-chemistryResults.xml
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="chemistryResults"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-clinpathRuns.xml b/PMR/resources/etls/prime-clinpathRuns.xml
index 42af3a095..70b9c6f04 100644
--- a/PMR/resources/etls/prime-clinpathRuns.xml
+++ b/PMR/resources/etls/prime-clinpathRuns.xml
@@ -41,6 +41,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="clinpathRuns"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
index 510f69be4..53d2bec58 100644
--- a/PMR/resources/etls/prime-hematologyResults.xml
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="hematologyResults"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-histology.xml b/PMR/resources/etls/prime-histology.xml
index 9f59b0b3b..b20efed03 100644
--- a/PMR/resources/etls/prime-histology.xml
+++ b/PMR/resources/etls/prime-histology.xml
@@ -39,6 +39,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="histology"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-microbiology.xml b/PMR/resources/etls/prime-microbiology.xml
index d49d51582..0f7a74d7b 100644
--- a/PMR/resources/etls/prime-microbiology.xml
+++ b/PMR/resources/etls/prime-microbiology.xml
@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="microbiology"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-pathologyDiagnoses.xml b/PMR/resources/etls/prime-pathologyDiagnoses.xml
index 172297dcb..aa9a38a63 100644
--- a/PMR/resources/etls/prime-pathologyDiagnoses.xml
+++ b/PMR/resources/etls/prime-pathologyDiagnoses.xml
@@ -38,6 +38,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="pathologyDiagnoses"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-weight.xml b/PMR/resources/etls/prime-weight.xml
index 435679e81..e08c00c11 100644
--- a/PMR/resources/etls/prime-weight.xml
+++ b/PMR/resources/etls/prime-weight.xml
@@ -37,6 +37,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="weight"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
index 95d0c2f2c..b1dcd8609 100644
--- a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
+++ b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
@@ -1,6 +1,7 @@
 package org.labkey.primeseq.etl;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
@@ -50,7 +51,8 @@ private enum Settings
         destSchema(true),
         destQuery(true),
         destColumn(true),
-        destAdditionalFilters(false);
+        destAdditionalFilters(false),
+        reportOnly(false);
 
         private final boolean _isRequired;
 
@@ -106,6 +108,11 @@ public void setSettings(Map<String, String> settings)
         _settings.putAll(settings);
     }
 
+    private boolean isReportOnly()
+    {
+        return _settings.containsKey(Settings.reportOnly.name()) && Boolean.parseBoolean(_settings.get(Settings.reportOnly.name()));
+    }
+
     private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Container c, Logger log) throws IllegalStateException
     {
         DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, c, log);
@@ -259,7 +266,13 @@ private void verifyRows(PipelineJob job) throws PipelineJobException
 
         if (source != dest)
         {
-            job.getLogger().error("Row counts do not match (source: {}, dest: {})!", source, dest);
+            if (isReportOnly()) {
+                job.getLogger().info("Row counts do not match (source: {}, dest: {})!", source, dest);
+            }
+            else
+            {
+                job.getLogger().error("Row counts do not match (source: {}, dest: {})!", source, dest);
+            }
         }
     }
 }

From 3c16a6e4e8b221bf07e66907aaddf3fa29329978 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 6 Oct 2025 12:42:24 -0700
Subject: [PATCH 26/40] Drop unused fields

---
 PMR/resources/etls/prime-chemistryResults.xml  | 1 -
 PMR/resources/etls/prime-hematologyResults.xml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
index 380c88796..bda1efe8d 100644
--- a/PMR/resources/etls/prime-chemistryResults.xml
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -8,7 +8,6 @@
                 <sourceColumns>
                     <column>Id</column>
                     <column>date</column>
-                    <column>ageAtTime</column>
                     <column>testId</column>
                     <column>result</column>
                     <column>units</column>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
index 53d2bec58..e3aca8f31 100644
--- a/PMR/resources/etls/prime-hematologyResults.xml
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -8,7 +8,6 @@
                 <sourceColumns>
                     <column>Id</column>
                     <column>date</column>
-                    <column>ageAtTime</column>
                     <column>testId</column>
                     <column>result</column>
                     <column>units</column>

From c2f37222dae672562e0b2cda2994f267b60c9adb Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 7 Oct 2025 10:28:22 -0700
Subject: [PATCH 27/40] Restore tighter validation

---
 PMR/resources/etls/prime-blooddraws.xml         | 2 +-
 PMR/resources/etls/prime-chemistryResults.xml   | 2 +-
 PMR/resources/etls/prime-clinpathRuns.xml       | 2 +-
 PMR/resources/etls/prime-hematologyResults.xml  | 2 +-
 PMR/resources/etls/prime-histology.xml          | 2 +-
 PMR/resources/etls/prime-microbiology.xml       | 2 +-
 PMR/resources/etls/prime-pathologyDiagnoses.xml | 5 +----
 PMR/resources/etls/prime-weight.xml             | 2 +-
 8 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/PMR/resources/etls/prime-blooddraws.xml b/PMR/resources/etls/prime-blooddraws.xml
index 93d4a6cdf..e9a49d6d6 100644
--- a/PMR/resources/etls/prime-blooddraws.xml
+++ b/PMR/resources/etls/prime-blooddraws.xml
@@ -35,7 +35,7 @@
                     <setting name="destQuery" value="blood"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
index bda1efe8d..10d318563 100644
--- a/PMR/resources/etls/prime-chemistryResults.xml
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -40,7 +40,7 @@
                     <setting name="destQuery" value="chemistryResults"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-clinpathRuns.xml b/PMR/resources/etls/prime-clinpathRuns.xml
index 70b9c6f04..d74ccc6e6 100644
--- a/PMR/resources/etls/prime-clinpathRuns.xml
+++ b/PMR/resources/etls/prime-clinpathRuns.xml
@@ -42,7 +42,7 @@
                     <setting name="destQuery" value="clinpathRuns"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
index e3aca8f31..4ba39f448 100644
--- a/PMR/resources/etls/prime-hematologyResults.xml
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -40,7 +40,7 @@
                     <setting name="destQuery" value="hematologyResults"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-histology.xml b/PMR/resources/etls/prime-histology.xml
index b20efed03..1986eed71 100644
--- a/PMR/resources/etls/prime-histology.xml
+++ b/PMR/resources/etls/prime-histology.xml
@@ -40,7 +40,7 @@
                     <setting name="destQuery" value="histology"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-microbiology.xml b/PMR/resources/etls/prime-microbiology.xml
index 0f7a74d7b..73122f386 100644
--- a/PMR/resources/etls/prime-microbiology.xml
+++ b/PMR/resources/etls/prime-microbiology.xml
@@ -41,7 +41,7 @@
                     <setting name="destQuery" value="microbiology"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-pathologyDiagnoses.xml b/PMR/resources/etls/prime-pathologyDiagnoses.xml
index aa9a38a63..efed3fb16 100644
--- a/PMR/resources/etls/prime-pathologyDiagnoses.xml
+++ b/PMR/resources/etls/prime-pathologyDiagnoses.xml
@@ -8,12 +8,9 @@
                 <sourceColumns>
                     <column>Id</column>
                     <column>date</column>
-                    <column>ageAtTime</column>
                     <column>sort_order</column>
                     <column>codes</column>
                     <column>objectid</column>
-                    <column>created</column>
-                    <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
                 <sourceFilters>
@@ -39,7 +36,7 @@
                     <setting name="destQuery" value="pathologyDiagnoses"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-weight.xml b/PMR/resources/etls/prime-weight.xml
index e08c00c11..fd651cf4d 100644
--- a/PMR/resources/etls/prime-weight.xml
+++ b/PMR/resources/etls/prime-weight.xml
@@ -38,7 +38,7 @@
                     <setting name="destQuery" value="weight"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>

From 6de45c5d77e355700a6382953c70caed562258f9 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 7 Oct 2025 15:34:34 -0700
Subject: [PATCH 28/40] Update dependencies

---
 mcc/package-lock.json | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/mcc/package-lock.json b/mcc/package-lock.json
index b20dd7fb3..a799d2232 100644
--- a/mcc/package-lock.json
+++ b/mcc/package-lock.json
@@ -8193,10 +8193,11 @@
       }
     },
     "node_modules/tar-fs": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz",
-      "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==",
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
+      "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",

From 8b980c1f849f300f9d63bb36150b5932d8f0e88f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 15 Oct 2025 11:02:16 -0700
Subject: [PATCH 29/40] Update MCC text

---
 mcc/resources/views/mccRequests.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mcc/resources/views/mccRequests.html b/mcc/resources/views/mccRequests.html
index 53e77dce9..f47a1ee71 100644
--- a/mcc/resources/views/mccRequests.html
+++ b/mcc/resources/views/mccRequests.html
@@ -8,7 +8,7 @@
 
         Ext4.get(webpart.wrapperDivId).update(
             'The MCC is now accepting animal requests! Investigators interested in requesting marmosets for their research can submit applications via the animal request portal.<br><br>' +
-            '<span style="font-weight: bold">Investigators can anticipate the estimated cost for getting marmosets to be about $5,500 USD per animal plus approximately $10,000 USD for shipping. Please note that this is an estimate and the actual cost is determined by the breeding centers and shipping can vary based on distance.</span>' +
+            '<span style="font-weight: bold">Investigators can anticipate the following estimated cost per animal: $5,500 USD for early-stage investigators, $6,500 USD for other academic investigators and $10,000 USD for commercial institutions. Arranging shipping is the responsibility of the requestor and is approximately $10,000 USD. Please note that this is an estimate and the actual cost is determined by the breeding centers and shipping can vary based on distance.</span>' +
             '<p></p>' +
             '<a class="labkey-text-link" href="<%=contextPath%>/mcc' + ctx['MCCRequestContainer'] + '/animalRequest.view">Submit New Animal Request</a><br>' +
             '<a class="labkey-text-link" href="/become-a-user.html#request-animals">Click Here to View Documentation on the Request Process and Scoring Criteria</a>' +

From 82c728eabf6f1355bbcd80439579a9d003575352 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 15 Oct 2025 14:02:42 -0700
Subject: [PATCH 30/40] Ensure extra memory for BWA-mem

---
 .../primeseq/pipeline/SequenceJobResourceAllocator.java      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
index c213266fd..cb15b069a 100644
--- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
+++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
@@ -96,6 +96,11 @@ private int getAlignerIndexMem(PipelineJob job)
                 }
             }
         }
+        else if (job.getClass().getName().endsWith("ReferenceLibraryPipelineJob"))
+        {
+            // This almost always includes bwa-mem
+            return 72;
+        }
 
         return 36;
     }

From d788191491866f93515b354cc07e94e2faec8566 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 25 Oct 2025 08:11:02 -0700
Subject: [PATCH 31/40] Increase memory for kinship task

---
 .../primeseq/pipeline/SequenceJobResourceAllocator.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
index cb15b069a..b7921c6a3 100644
--- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
+++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
@@ -184,8 +184,8 @@ public Integer getMaxRequestMemory(PipelineJob job)
 
         if (isGeneticsTask(job))
         {
-            job.getLogger().debug("setting memory to 72");
-            return 72;
+            job.getLogger().debug("setting memory to 96");
+            return 96;
         }
 
         if (isCacheAlignerIndexesTask(job))

From 084f20f9280370ffa1d9c517574c602638930587 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 14:07:25 -0700
Subject: [PATCH 32/40] Misc small improvements to MCC

---
 mcc/resources/etls/mcc.xml                    |  1 +
 mcc/resources/etls/snprc-datasets.xml         |  2 +-
 .../MCC NIH Dashboard.folderType.xml          | 15 +++++
 .../queries/mcc/genomicDatasetsSource.sql     |  3 +-
 .../queries/study/genomicDatasets/.qview.xml  |  1 +
 .../study/datasets/datasets_metadata.xml      |  3 +
 mcc/resources/views/mccU24Demographics.html   |  5 +-
 mcc/src/client/Dashboard/Dashboard.tsx        |  4 +-
 mcc/src/client/GeneticsPlot/GeneticsPlot.tsx  | 61 +++++++++----------
 mcc/src/client/GeneticsPlot/KinshipTable.tsx  | 11 ++++
 mcc/src/client/GeneticsPlot/ScatterChart.tsx  | 11 ++++
 .../client/GeneticsPlot/SequenceDataTable.tsx | 39 ++++++++++++
 mcc/src/org/labkey/mcc/MccUserSchema.java     |  1 +
 .../mcc/etl/PopulateGeneticDataStep.java      |  3 +-
 14 files changed, 123 insertions(+), 37 deletions(-)
 create mode 100644 mcc/src/client/GeneticsPlot/SequenceDataTable.tsx

diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml
index 9e3d71ad9..d324ce40e 100644
--- a/mcc/resources/etls/mcc.xml
+++ b/mcc/resources/etls/mcc.xml
@@ -86,6 +86,7 @@
                     <column>date</column>
                     <column>datatype</column>
                     <column>sra_accession</column>
+                    <column>total_reads</column>
                     <column>objectid</column>
                 </sourceColumns>
             </source>
diff --git a/mcc/resources/etls/snprc-datasets.xml b/mcc/resources/etls/snprc-datasets.xml
index 9b66a4814..350a55d38 100644
--- a/mcc/resources/etls/snprc-datasets.xml
+++ b/mcc/resources/etls/snprc-datasets.xml
@@ -155,7 +155,7 @@
         <transform id="observations1" type="RemoteQueryTransformStep">
             <description>Copy to target</description>
             <source remoteSource="SNPRC" schemaName="study" queryName="u24MarmosetStats">
-                <!-- NOTE: the column orde must be preserved and match expected order in NprcObservationStep -->
+                <!-- NOTE: the column order must be preserved and match expected order in NprcObservationStep -->
                 <sourceColumns>
                     <column>AnimalId</column>
                     <column>Date</column>
diff --git a/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml b/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml
index 8bf4b4864..9bea3b2ea 100644
--- a/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml	
+++ b/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml	
@@ -74,6 +74,21 @@
                 </webPart>
             </preferredWebParts>
         </folderTab>
+        <folderTab>
+            <name>geneticsDashboard</name>
+            <caption>Genetics</caption>
+            <selectors>
+                <selector>
+                    <view>genetics</view>
+                </selector>
+            </selectors>
+            <preferredWebParts>
+                <webPart>
+                    <name>geneticsPlotWebpart</name>
+                    <location>body</location>
+                </webPart>
+            </preferredWebParts>
+        </folderTab>
         <folderTab>
             <name>requests</name>
             <caption>MCC Requests</caption>
diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.sql b/mcc/resources/queries/mcc/genomicDatasetsSource.sql
index 1dcdba771..332e76afc 100644
--- a/mcc/resources/queries/mcc/genomicDatasetsSource.sql
+++ b/mcc/resources/queries/mcc/genomicDatasetsSource.sql
@@ -3,7 +3,8 @@ SELECT
     r.subjectid as Id,
     r.created as date,
     r.application as datatype,
-    r.sraRuns as sra_accession
+    r.sraRuns as sra_accession,
+    r.totalForwardReads as total_reads
 
 FROM sequenceanalysis.sequence_readsets r
 
diff --git a/mcc/resources/queries/study/genomicDatasets/.qview.xml b/mcc/resources/queries/study/genomicDatasets/.qview.xml
index e583f4f41..c29532b3c 100644
--- a/mcc/resources/queries/study/genomicDatasets/.qview.xml
+++ b/mcc/resources/queries/study/genomicDatasets/.qview.xml
@@ -4,6 +4,7 @@
        <column name="date" />
        <column name="datatype" />
        <column name="sra_accession" />
+       <column name="total_reads" />
        <column name="history" />
    </columns>
    <sorts>
diff --git a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml
index d1ce17b9d..60b6cda93 100644
--- a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -1073,6 +1073,9 @@
             <column columnName="sra_accession">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="total_reads">
+                <datatype>integer</datatype>
+            </column>
             <column columnName="objectid">
                 <datatype>entityid</datatype>
                 <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
diff --git a/mcc/resources/views/mccU24Demographics.html b/mcc/resources/views/mccU24Demographics.html
index e490ff8d5..5afcd1cd4 100644
--- a/mcc/resources/views/mccU24Demographics.html
+++ b/mcc/resources/views/mccU24Demographics.html
@@ -12,7 +12,10 @@
         title: 'MCC Animals',
         schemaName: 'study',
         queryName: 'demographics',
-        viewName: LABKEY.ActionURL.getParameter('viewName') ?? 'U24 Assigned'
+        viewName: LABKEY.ActionURL.getParameter('viewName'),
+        removeableFilters: [
+            LABKEY.Filter.create('u24_status', true, LABKEY.Filter.Types.EQUALS)
+        ]
     }).render(webpart.wrapperDivId);
 });
 
diff --git a/mcc/src/client/Dashboard/Dashboard.tsx b/mcc/src/client/Dashboard/Dashboard.tsx
index fe1ba520d..382e95add 100644
--- a/mcc/src/client/Dashboard/Dashboard.tsx
+++ b/mcc/src/client/Dashboard/Dashboard.tsx
@@ -88,9 +88,9 @@ export function Dashboard() {
                 </div>
                 <div className="col-md-4">
                     <div className="panel panel-default">
-                        <div className="panel-heading">Center (All Animals)</div>
+                        <div className="panel-heading">Center (Living Animals)</div>
                         <div className="panel-body">
-                            <PieChart fieldName = "colony" demographics={demographics} cutout = "30%" />
+                            <PieChart fieldName = "colony" demographics={living} cutout = "30%" />
                         </div>
                     </div>
                 </div>
diff --git a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
index 031af83d2..b33734bbb 100644
--- a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
+++ b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
@@ -5,21 +5,13 @@ import ScatterChart from './ScatterChart';
 import { Box, Tab, Tabs } from '@mui/material';
 import KinshipTable from './KinshipTable';
 import { ErrorBoundary } from '../components/ErrorBoundary';
+import SequenceDataTable from './SequenceDataTable';
 
 
-function GenomeBrowser(props: {jbrowseId: any}) {
-    const { jbrowseId } = props;
-
-    return (
-        <div>
-            <a href={ActionURL.buildURL('jbrowse', 'jbrowse', null, {session: jbrowseId})}>Click here to view Marmoset SNP data in the genome browser</a>
-        </div>
-    );
-}
-
 export function GeneticsPlot() {
     const [pcaData, setPcaData] = useState([]);
     const [kinshipData, setKinshipData] = useState([]);
+    const [sequenceData, setSequenceData] = useState([]);
     const [jbrowseId, setJBrowseId] = useState(null);
     const [value, setValue] = React.useState(0);
 
@@ -83,6 +75,29 @@ export function GeneticsPlot() {
             },
             scope: this
         });
+
+        Query.selectRows({
+            containerPath: containerPath,
+            schemaName: 'study',
+            queryName: 'genomicDatasets',
+            columns: 'Id,datatype,sra_accession,total_reads,objectid',
+            success: function(results) {
+                setSequenceData(results.rows.map((row) => {
+                    return({
+                        id: row.objectid,
+                        Id: row.Id,
+                        datatype: row.datatype,
+                        sra_accession: row.sra_accession,
+                        total_reads: row.total_reads
+                    })
+                }))
+            },
+            failure: function(response) {
+                alert('There was an error loading data');
+                console.log(response);
+            },
+            scope: this
+        });
     }, [] /* only run the effect on mount */);
 
     if (!containerPath) {
@@ -113,35 +128,19 @@ export function GeneticsPlot() {
                 578 marmosets on NCBI's Sequence Read Archive (SRA), we are excited to report that the MCC portal now
                 houses a call set with single nucleotide variants and short indels for over 800 individuals.
                 <p/>
-                The MCC genomic database is extensive, with each individual being genotype at millions of variants
-                across the genome. One way to summarize a large dataset can be done using Principal Component Analysis
-                (PCA). PCA is a technique used across disciplines (from astronomy to genomics) that reduces the
-                information in a multi-dimensional dataset to (fewer) principal components (PC) that retain overall
-                trends and patterns in the original data. Biologically, this could mean merging together two variants
-                that are always inherited together into just one PC, making the data easier to analyze while maintaining
-                its most important patterns. See the **Visualization with PCA** tab below.
-                <p/>
-                Although PCA is useful for broad-scale comparisons, it is not very useful when trying to distinguish
-                whether two individuals are siblings or first-cousins, for instance. For that, we have better statistics
-                that can describe the genetic relatedness between two individuals. We estimated genetic relatedness for
-                all pairs of individuals for which we have whole-genome data, and made these available under the
-                **Kinship** tab. There you will find the inferred relationships between pairs of individuals as well as
-                the calculated kinship coefficient, which is a quantitative measure of genetic relatedness
-                (see <a href="https://en.wikipedia.org/wiki/Coefficient_of_relationship#Kinship_coefficient">here</a> for more details).
-                <p/>
-                It is possible to explore the full MCC database of variants with a graphical interface by accessing the
-                **Genome Browser** tab. There you can, for example, visualize all the variants present in your gene of
-                interest by typing it's name in the search bar.
+                In addition to the information in the tabs below, you can use the MCC genome browser to view know variants and search by gene.
+                <a href={ActionURL.buildURL('jbrowse', 'jbrowse', null, {session: jbrowseId})}>Click here to view Marmoset SNP data in the genome browser</a>
                 <p/>
                 The genetic analyses described here were performed by Karina Ray (ONPRC), Murillo Rodrigues (ONPRC), and
                 Ric del Rosario (Broad Institute). Please contact us at <a href="mailto:mcc@ohsu.edu">mcc@ohsu.edu</a> with any
                 questions.
             </div>
+
             <Box sx={{ borderBottom: 1, borderColor: 'divider' }}>
                 <Tabs value={value} onChange={handleChange} aria-label="basic tabs example">
                     <Tab label="Population Genetic Diversity" {...a11yProps(0)} />
                     <Tab label="Kinship" {...a11yProps(1)} />
-                    <Tab label="Genetic Variants" {...a11yProps(2)} hidden={jbrowseId == null}/>
+                    <Tab label="Sequence Datasets" {...a11yProps(2)}/>
                 </Tabs>
             </Box>
             <div className="row">
@@ -150,7 +149,7 @@ export function GeneticsPlot() {
                         <div className="panel-body">
                             {value === 0 && <ScatterChart data={pcaData}/>}
                             {value === 1 && <KinshipTable data={kinshipData}/>}
-                            {value === 2 && <GenomeBrowser jbrowseId={jbrowseId}/>}
+                            {value === 2 && <SequenceDataTable data={sequenceData}/>}
                         </div>
                     </div>
                 </div>
diff --git a/mcc/src/client/GeneticsPlot/KinshipTable.tsx b/mcc/src/client/GeneticsPlot/KinshipTable.tsx
index ee4f316fa..a95052ce1 100644
--- a/mcc/src/client/GeneticsPlot/KinshipTable.tsx
+++ b/mcc/src/client/GeneticsPlot/KinshipTable.tsx
@@ -13,6 +13,16 @@ export default function KinshipTable(props: {data: any}) {
     ]
 
     return (
+        <>
+        <div style={{paddingBottom: 20, maxWidth: 1000}}>
+            Although PCA is useful for broad-scale comparisons, it is not very useful when trying to distinguish
+            whether two individuals are siblings or first-cousins, for instance. For that, we have better statistics
+            that can describe the genetic relatedness between two individuals. We estimated genetic relatedness for
+            all pairs of individuals for which we have whole-genome data, shown in the table below. There you will
+            find the inferred relationships between pairs of individuals as well as the calculated kinship coefficient,
+            which is a quantitative measure of genetic relatedness
+            (see <a href="https://en.wikipedia.org/wiki/Coefficient_of_relationship#Kinship_coefficient">here</a> for more details).
+        </div>
         <DataGrid
             autoHeight={true}
             columns={columns}
@@ -24,5 +34,6 @@ export default function KinshipTable(props: {data: any}) {
             paginationModel={pageModel}
             onPaginationModelChange={(model) => setPageModel(model)}
         />
+        </>
     );
 }
\ No newline at end of file
diff --git a/mcc/src/client/GeneticsPlot/ScatterChart.tsx b/mcc/src/client/GeneticsPlot/ScatterChart.tsx
index 3ffd23e34..1439e3584 100644
--- a/mcc/src/client/GeneticsPlot/ScatterChart.tsx
+++ b/mcc/src/client/GeneticsPlot/ScatterChart.tsx
@@ -82,6 +82,17 @@ export default function ScatterChart(props: {data: any}) {
     }
 
     return (
+        <>
+        <div style={{paddingBottom: 20, maxWidth: 1000}}>
+            The MCC genomic database is extensive, with each individual being genotype at millions of variants
+            across the genome. One way to summarize a large dataset can be done using Principal Component Analysis
+            (PCA). PCA is a technique used across disciplines (from astronomy to genomics) that reduces the
+            information in a multi-dimensional dataset to (fewer) principal components (PC) that retain overall
+            trends and patterns in the original data. Biologically, this could mean merging together two variants
+            that are always inherited together into just one PC, making the data easier to analyze while maintaining
+            its most important patterns.
+        </div>
         <Scatter data={chartData} options={chartOptions}/>
+        </>
     );
 }
\ No newline at end of file
diff --git a/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx b/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
new file mode 100644
index 000000000..6d137b4ca
--- /dev/null
+++ b/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
@@ -0,0 +1,39 @@
+import React from 'react';
+import { DataGrid, GridColDef, GridPaginationModel, GridRenderCellParams, GridToolbar } from '@mui/x-data-grid';
+
+export default function SequenceDataTable(props: {data: any}) {
+    const { data } = props;
+    const [pageModel, setPageModel] = React.useState<GridPaginationModel>({page: 0, pageSize: 25});
+
+    const columns: GridColDef[] = [
+        { field: 'Id', headerName: 'Animal 1', width: 150, type: "string", headerAlign: 'left' },
+        { field: 'datatype', headerName: 'Datatype', width: 250, type: "string", headerAlign: 'left' },
+        { field: 'sra_accession', headerName: 'SRA Accession', width: 150, type: "string", headerAlign: 'right', renderCell: (params: GridRenderCellParams<any, string>) => {
+                return (
+                    <a
+                        target="_blank"
+                        href={params.value}
+                    >{params.value ? "https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=" + params.value : ""}</a>
+                );
+            }},
+        { field: 'total_reads', headerName: 'Total Reads', width: 125, type: "number", headerAlign: 'left', flex: 1 }
+    ]
+
+    return (
+        <>
+        <div style={{paddingBottom: 20, maxWidth: 1000}}>
+        </div>
+        <DataGrid
+            autoHeight={true}
+            columns={columns}
+            rows={data}
+            slots={{
+                toolbar: GridToolbar
+            }}
+            pageSizeOptions={[10,25,50,100]}
+            paginationModel={pageModel}
+            onPaginationModelChange={(model) => setPageModel(model)}
+        />
+        </>
+    );
+}
\ No newline at end of file
diff --git a/mcc/src/org/labkey/mcc/MccUserSchema.java b/mcc/src/org/labkey/mcc/MccUserSchema.java
index 004814851..71912fc5d 100644
--- a/mcc/src/org/labkey/mcc/MccUserSchema.java
+++ b/mcc/src/org/labkey/mcc/MccUserSchema.java
@@ -271,6 +271,7 @@ private TableInfo getGenomicsQuery()
                 "    d.date,\n" +
                 "    d.datatype,\n" +
                 "    d.sra_accession,\n" +
+                "    d.total_reads,\n" +
                 "    d.objectid,\n" +
                 "    d.container\n" +
                 "\n" +
diff --git a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java
index 9e27a7c4e..06feea72d 100644
--- a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java
+++ b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java
@@ -68,7 +68,7 @@ private void populateGeneticData(PipelineJob job) throws PipelineJobException
         {
             //first select all rows from remote table
             SelectRowsCommand sr = new SelectRowsCommand(MccSchema.NAME, "genomicDatasetsSource");
-            sr.setColumns(Arrays.asList("Id", "date", "datatype", "sra_accession"));
+            sr.setColumns(Arrays.asList("Id", "date", "datatype", "sra_accession", "total_reads"));
 
             TableInfo aggregatedDemographics = QueryService.get().getUserSchema(job.getUser(), job.getContainer(), MccSchema.NAME).getTable("aggregatedDemographics");
 
@@ -104,6 +104,7 @@ private void populateGeneticData(PipelineJob job) throws PipelineJobException
                 newRow.put("date", x.get("date"));
                 newRow.put("datatype", x.get("datatype"));
                 newRow.put("sra_accession", x.get("sra_accession"));
+                newRow.put("total_reads", x.get("total_reads"));
 
                 toInsert.get(target).add(newRow);
             });

From 1f467b67a53fe56237ca0b659c09e2ec4a34ce2f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 14:40:11 -0700
Subject: [PATCH 33/40] bugfix to url

---
 mcc/src/client/GeneticsPlot/SequenceDataTable.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx b/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
index 6d137b4ca..6ec8b2f35 100644
--- a/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
+++ b/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
@@ -12,8 +12,8 @@ export default function SequenceDataTable(props: {data: any}) {
                 return (
                     <a
                         target="_blank"
-                        href={params.value}
-                    >{params.value ? "https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=" + params.value : ""}</a>
+                        href={params.value ? "https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=" + params.value : ""}
+                    >{params.value}</a>
                 );
             }},
         { field: 'total_reads', headerName: 'Total Reads', width: 125, type: "number", headerAlign: 'left', flex: 1 }

From 1fb01f72b0d85ebbaca8d7165644a480a54f98a4 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 15:13:07 -0700
Subject: [PATCH 34/40] MCC webpart syntax fix

---
 mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml | 2 +-
 mcc/resources/queries/study/genomicDatasets.query.xml      | 4 ++++
 mcc/src/client/GeneticsPlot/webpart/app.tsx                | 2 +-
 mcc/src/client/GeneticsPlot/webpart/dev.tsx                | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml b/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml
index 9bea3b2ea..ae9d9e26c 100644
--- a/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml	
+++ b/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml	
@@ -84,7 +84,7 @@
             </selectors>
             <preferredWebParts>
                 <webPart>
-                    <name>geneticsPlotWebpart</name>
+                    <name>Marmoset Genetics</name>
                     <location>body</location>
                 </webPart>
             </preferredWebParts>
diff --git a/mcc/resources/queries/study/genomicDatasets.query.xml b/mcc/resources/queries/study/genomicDatasets.query.xml
index c78b85463..fd9406c2c 100644
--- a/mcc/resources/queries/study/genomicDatasets.query.xml
+++ b/mcc/resources/queries/study/genomicDatasets.query.xml
@@ -18,6 +18,10 @@
                         <columnTitle>SRA Accession</columnTitle>
                         <url>https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=${sra_accession}</url>
                     </column>
+                    <column columnName="total_reads">
+                        <columnTitle>Total Reads</columnTitle>
+                        <formatString>#,##0.##</formatString>
+                    </column>
                 </columns>
             </table>
         </tables>
diff --git a/mcc/src/client/GeneticsPlot/webpart/app.tsx b/mcc/src/client/GeneticsPlot/webpart/app.tsx
index 0d9235996..4b95f6c8f 100644
--- a/mcc/src/client/GeneticsPlot/webpart/app.tsx
+++ b/mcc/src/client/GeneticsPlot/webpart/app.tsx
@@ -4,6 +4,6 @@ import { App } from '@labkey/api';
 
 import { GeneticsPlot } from '../GeneticsPlot';
 
-App.registerApp<any>('mccPcaWebpart', target => {
+App.registerApp<any>('geneticsPlotWebpart', target => {
     ReactDOM.render(<GeneticsPlot />, document.getElementById(target));
 });
diff --git a/mcc/src/client/GeneticsPlot/webpart/dev.tsx b/mcc/src/client/GeneticsPlot/webpart/dev.tsx
index a503bfdbc..b49820176 100644
--- a/mcc/src/client/GeneticsPlot/webpart/dev.tsx
+++ b/mcc/src/client/GeneticsPlot/webpart/dev.tsx
@@ -4,6 +4,6 @@ import { App } from '@labkey/api';
 
 import { GeneticsPlot } from '../GeneticsPlot';
 
-App.registerApp<any>('mccPcaWebpart', target => {
+App.registerApp<any>('geneticsPlotWebpart', target => {
     ReactDOM.render(<GeneticsPlot />, document.getElementById(target));
 }, true);
\ No newline at end of file

From 7916bd8eff618867e1b1d9e89adee07bf51cac27 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 15:44:48 -0700
Subject: [PATCH 35/40] MCC webpart syntax fix

---
 mcc/src/client/GeneticsPlot/GeneticsPlot.tsx | 20 ++++++++++++++------
 mcc/src/client/entryPoints.js                |  5 +++--
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
index b33734bbb..6a7908211 100644
--- a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
+++ b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
@@ -126,14 +126,22 @@ export function GeneticsPlot() {
                 Over the past few years, the MCC team has been working on extracting, sequencing and analyzing DNA from
                 marmosets across the participating breeding centers. While we have deposited the raw sequence data for
                 578 marmosets on NCBI's Sequence Read Archive (SRA), we are excited to report that the MCC portal now
-                houses a call set with single nucleotide variants and short indels for over 800 individuals.
-                <p/>
-                In addition to the information in the tabs below, you can use the MCC genome browser to view know variants and search by gene.
-                <a href={ActionURL.buildURL('jbrowse', 'jbrowse', null, {session: jbrowseId})}>Click here to view Marmoset SNP data in the genome browser</a>
-                <p/>
-                The genetic analyses described here were performed by Karina Ray (ONPRC), Murillo Rodrigues (ONPRC), and
+                houses a call set with single nucleotide variants and short indels for over 800 individuals. The genetic analyses
+                described here were performed by Karina Ray (ONPRC), Murillo Rodrigues (ONPRC), and
                 Ric del Rosario (Broad Institute). Please contact us at <a href="mailto:mcc@ohsu.edu">mcc@ohsu.edu</a> with any
                 questions.
+                <p/>
+                { jbrowseId ? (
+                    <>
+                        In addition to the information in the tabs below, you can use the MCC genome browser to view variants and/or search by gene:
+                        <p/>
+                        <ul>
+                            <li>
+                                <a style={{fontWeight: 'bold'}} href={ActionURL.buildURL('jbrowse', 'jbrowse', null, {session: jbrowseId})}>Click here to open the genome browser</a>
+                            </li>
+                        </ul>
+                    </>
+                ) : null }
             </div>
 
             <Box sx={{ borderBottom: 1, borderColor: 'divider' }}>
diff --git a/mcc/src/client/entryPoints.js b/mcc/src/client/entryPoints.js
index 9063de69b..2f2cf21e4 100644
--- a/mcc/src/client/entryPoints.js
+++ b/mcc/src/client/entryPoints.js
@@ -24,12 +24,13 @@ module.exports = {
     name: 'geneticsPlot',
     title: 'Marmoset Genetics',
     permissionClasses: ['org.labkey.api.security.permissions.ReadPermission'],
-    path: './src/client/GeneticsPlot'
+    path: './src/client/GeneticsPlot',
   }, {
     name: 'geneticsPlotWebpart',
     title: 'Marmoset Genetics',
     permissionClasses: ['org.labkey.api.security.permissions.ReadPermission'],
-    path: './src/client/GeneticsPlot/webpart'
+    path: './src/client/GeneticsPlot/webpart',
+    generateLib: true
   },{
     name: 'u24Dashboard',
     title: 'U24 Dashboard',

From c1f3405a3be119a5d1c2772819400ec244415074 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 15:57:49 -0700
Subject: [PATCH 36/40] Add number formatting

---
 mcc/src/client/Dashboard/Dashboard.tsx    | 6 +++---
 mcc/src/client/U24Dashboard/Dashboard.tsx | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/mcc/src/client/Dashboard/Dashboard.tsx b/mcc/src/client/Dashboard/Dashboard.tsx
index 382e95add..873da1123 100644
--- a/mcc/src/client/Dashboard/Dashboard.tsx
+++ b/mcc/src/client/Dashboard/Dashboard.tsx
@@ -66,20 +66,20 @@ export function Dashboard() {
                         <div className="panel-heading">Census</div>
                         <div className="row">
                             <div className="panel-body count-panel-body">
-                                <div className="count-panel-text">{demographics.length}</div>
+                                <div className="count-panel-text">{new Intl.NumberFormat("en-IN").format(demographics.length)}</div>
                                 <div className="small text-muted">Marmosets tracked by MCC</div>
                             </div>
                         </div>
                         <div className="row mcc-col-centered">
                             <div className="col-md-3">
                                 <div className="panel-body count-panel-body">
-                                    <div className="count-panel-text-small">{living.length}</div>
+                                    <div className="count-panel-text-small">{new Intl.NumberFormat("en-IN").format(living.length)}</div>
                                     <div className="small text-muted text-center">Living</div>
                                 </div>
                             </div>
                             <div className="col-md-3">
                                 <div className="panel-body count-panel-body">
-                                    <div className="count-panel-text-small">{u24Assigned.length}</div>
+                                    <div className="count-panel-text-small">{new Intl.NumberFormat("en-IN").format(u24Assigned.length)}</div>
                                     <div className="small text-muted text-center">U24 Assigned</div>
                                 </div>
                             </div>
diff --git a/mcc/src/client/U24Dashboard/Dashboard.tsx b/mcc/src/client/U24Dashboard/Dashboard.tsx
index 8819a0789..513ae0faa 100644
--- a/mcc/src/client/U24Dashboard/Dashboard.tsx
+++ b/mcc/src/client/U24Dashboard/Dashboard.tsx
@@ -147,20 +147,20 @@ export function Dashboard() {
                         <div className="panel-heading">U24 Census</div>
                         <div className="row">
                             <div className="panel-body count-panel-body">
-                                <div className="count-panel-text"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "animalData"})}>{u24Assigned.length}</a></div>
+                                <div className="count-panel-text"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "animalData"})}>{new Intl.NumberFormat("en-IN").format(u24Assigned.length)}</a></div>
                                 <div className="small text-muted text-center">Total U24 Animals</div>
                             </div>
                         </div>
                         <div className="row mcc-col-centered">
                             <div className="col-md-3">
                                 <div className="panel-body count-panel-body">
-                                    <div className="count-panel-text-small"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "animalData", "u24.Availability~eq": "available for transfer"})}>{availableForTransfer.length}</a></div>
+                                    <div className="count-panel-text-small"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "animalData", "u24.Availability~eq": "available for transfer"})}>{new Intl.NumberFormat("en-IN").format(availableForTransfer.length)}</a></div>
                                     <div className="small text-muted text-center">Available</div>
                                 </div>
                             </div>
                             <div className="col-md-3">
                                 <div className="panel-body count-panel-body">
-                                    <div className="count-panel-text-small"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "requests"})}>{requestRows.length}</a></div>
+                                    <div className="count-panel-text-small"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "requests"})}>{new Intl.NumberFormat("en-IN").format(requestRows.length)}</a></div>
                                     <div className="small text-muted text-center">Total Requests</div>
                                 </div>
                             </div>

From 95c14a3a998d4ef52edb5e06c5f1f0ba7b49bdec Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 17:01:56 -0700
Subject: [PATCH 37/40] Collapse small colonies

---
 mcc/src/client/Dashboard/Dashboard.tsx        |  8 ++--
 mcc/src/client/U24Dashboard/Dashboard.tsx     | 16 +++----
 .../client/components/dashboard/PieChart.tsx  | 42 +++++++++++++------
 3 files changed, 41 insertions(+), 25 deletions(-)

diff --git a/mcc/src/client/Dashboard/Dashboard.tsx b/mcc/src/client/Dashboard/Dashboard.tsx
index 873da1123..f9a2ba14a 100644
--- a/mcc/src/client/Dashboard/Dashboard.tsx
+++ b/mcc/src/client/Dashboard/Dashboard.tsx
@@ -8,9 +8,9 @@ import PieChart from '../components/dashboard/PieChart';
 import BarChart from '../components/dashboard/BarChart';
 
 export function Dashboard() {
-    const [demographics, setDemographics] = useState(null);
-    const [living, setLiving] = useState(null);
-    const [u24Assigned, setu24Assigned] = useState(null);
+    const [demographics, setDemographics] = useState<[]>(null);
+    const [living, setLiving] = useState<[]>(null);
+    const [u24Assigned, setu24Assigned] = useState<[]>(null);
 
     const ctx = getServerContext().getModuleContext('mcc') || {};
     const containerPath = ctx.MCCContainer || null;
@@ -90,7 +90,7 @@ export function Dashboard() {
                     <div className="panel panel-default">
                         <div className="panel-heading">Center (Living Animals)</div>
                         <div className="panel-body">
-                            <PieChart fieldName = "colony" demographics={living} cutout = "30%" />
+                            <PieChart fieldName = "colony" demographics={living} cutout = "30%" collapseBelow = {0.025}  />
                         </div>
                     </div>
                 </div>
diff --git a/mcc/src/client/U24Dashboard/Dashboard.tsx b/mcc/src/client/U24Dashboard/Dashboard.tsx
index 513ae0faa..7ade780c7 100644
--- a/mcc/src/client/U24Dashboard/Dashboard.tsx
+++ b/mcc/src/client/U24Dashboard/Dashboard.tsx
@@ -8,14 +8,14 @@ import BarChart, { ColorType } from '../components/dashboard/BarChart';
 import { ActiveElement, Chart, ChartEvent } from 'chart.js/dist/types/index';
 
 export function Dashboard() {
-    const [demographics, setDemographics] = useState(null);
-    const [living, setLiving] = useState(null);
-    const [u24Assigned, setu24Assigned] = useState(null);
-    const [availableForTransfer, setAvailableForTransfer] = useState(null);
-    const [requestRows, setRequestRows] = useState(null);
-    const [censusRows, setCensusRows] = useState(null);
-    const [birthData, setBirthData ] = useState(null);
-    const [breedingPairData, setBreedingPairData ] = useState(null);
+    const [demographics, setDemographics] = useState<[]>(null);
+    const [living, setLiving] = useState<[]>(null);
+    const [u24Assigned, setu24Assigned] = useState<[]>(null);
+    const [availableForTransfer, setAvailableForTransfer] = useState<[]>(null);
+    const [requestRows, setRequestRows] = useState<[]>(null);
+    const [censusRows, setCensusRows] = useState<[]>(null);
+    const [birthData, setBirthData ] = useState<[]>(null);
+    const [breedingPairData, setBreedingPairData ] = useState<[]>(null);
 
     const ctx = getServerContext().getModuleContext('mcc') || {};
     const containerPath = ctx.MCCContainer || null;
diff --git a/mcc/src/client/components/dashboard/PieChart.tsx b/mcc/src/client/components/dashboard/PieChart.tsx
index b9a28ec86..ca6880b4b 100644
--- a/mcc/src/client/components/dashboard/PieChart.tsx
+++ b/mcc/src/client/components/dashboard/PieChart.tsx
@@ -1,11 +1,5 @@
 import React, { useEffect, useRef } from 'react';
-import {
-    Chart,
-    ArcElement,
-    Legend,
-    PieController,
-    Tooltip
-} from 'chart.js';
+import { ArcElement, Chart, Legend, PieController, Tooltip } from 'chart.js';
 
 Chart.register(ArcElement, Legend, PieController, Tooltip);
 
@@ -21,14 +15,12 @@ const colors = [
     "#999999"
 ];
 
-export default function PieChart(props) {
+export default function PieChart(props: {demographics: [], fieldName: string, cutout?: string, collapseBelow?: number }) {
     const canvas = useRef(null);
 
-    const { demographics } = props;
-    const { fieldName } = props;
-    const { cutout } = props || 0;
+    const { demographics, fieldName, cutout = '0', collapseBelow = 0 } = props;
 
-    const collectedData = demographics.reduce((acc, curr) => {
+    const collectedData  = demographics.reduce((acc, curr) => {
         const value = curr[fieldName] === null ? 'Unknown' : curr[fieldName];
         if (acc[value]) {
             acc[value] = acc[value] + 1;
@@ -37,7 +29,31 @@ export default function PieChart(props) {
         }
 
         return acc;
-    }, {});
+    }, new Map<string, bigint>())
+
+    if (collapseBelow) {
+        const total = Object.keys(collectedData).reduce((sum, keyName) => {
+            sum += collectedData[keyName]
+
+            return sum
+        }, 0)
+
+        const otherValue = Object.keys(collectedData).reduce((sum, keyName) => {
+            const val = collectedData[keyName]
+            const fraction = val / total
+            if (fraction < collapseBelow) {
+                delete collectedData[keyName]
+                sum += val
+            }
+
+            return sum
+        }, 0)
+
+        if (otherValue) {
+            collectedData['Other'] = otherValue
+        }
+    }
+
     const labels = Object.keys(collectedData).sort(Intl.Collator().compare);
     const data = labels.map(label => collectedData[label]);
 

From e6b35986f0afc00f2b0becc4fdcbe35b7c437b0d Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sun, 2 Nov 2025 05:51:29 -0800
Subject: [PATCH 38/40] Improve behavior of MCC MarkShippedWindow

---
 .../web/mcc/window/MarkShippedWindow.js       | 343 +++++++++++-------
 1 file changed, 214 insertions(+), 129 deletions(-)

diff --git a/mcc/resources/web/mcc/window/MarkShippedWindow.js b/mcc/resources/web/mcc/window/MarkShippedWindow.js
index afa44eaad..a38fc3abc 100644
--- a/mcc/resources/web/mcc/window/MarkShippedWindow.js
+++ b/mcc/resources/web/mcc/window/MarkShippedWindow.js
@@ -208,7 +208,6 @@ Ext4.define('MCC.window.MarkShippedWindow', {
             return;
         }
 
-        var targetFolderId = win.down('#targetFolder').store.findRecord('Path', targetFolder).get('EntityId');
         Ext4.Msg.wait('Saving...');
         LABKEY.Query.selectRows({
             schemaName: 'study',
@@ -224,143 +223,229 @@ Ext4.define('MCC.window.MarkShippedWindow', {
                     return false;
                 }
 
-                var commands = [];
-                Ext4.Array.forEach(results.rows, function(row){
-                    var effectiveId = win.down('#usePreviousId-' + row.Id).getValue() ? row.Id : win.down('#newId-' + row.Id).getValue();
-                    var requestId = win.down('#requestId-' + row.Id).getValue();
-                    // This should be checked above, although perhaps case sensitivity could get involved:
-                    LDK.Assert.assertNotEmpty('Missing effective ID after query', effectiveId);
-
-                    var shouldAddDeparture = !row['Id/MostRecentDeparture/MostRecentDeparture'] ||
-                            row['Id/MostRecentDeparture/MostRecentDeparture'] !== Ext4.Date.format(row.effectiveDate, 'Y-m-d') ||
-                            row['Id/MostRecentDeparture/mccRequestId'] !== requestId ||
-                            row.Id !== effectiveId;
-                    if (shouldAddDeparture) {
-                        commands.push({
-                            command: 'insert',
-                            schemaName: 'study',
-                            queryName: 'Departure',
-                            rows: [{
-                                Id: row.Id,
-                                date: effectiveDate,
-                                source: row.colony,
-                                destination: centerName,
-                                mccRequestId: requestId,
-                                description: row.colony ? 'Original center: ' + row.colony : null,
-                                qcstate: null,
-                                objectId: null,
-                                QCStateLabel: 'Completed'
-                            }]
-                        });
-                    }
+                var uniqueIds = [];
+                Ext4.Array.forEach(results.rows, function(row) {
+                    uniqueIds.push(win.down('#usePreviousId-' + row.Id).getValue() ? row.Id : win.down('#newId-' + row.Id).getValue());
+                }, this);
 
-                    // If going to a new LK folder, we're creating a whole new record:
-                    if (targetFolderId.toUpperCase() !== LABKEY.Security.currentContainer.id.toUpperCase() || effectiveId !== row.Id) {
-                        commands.push({
-                            command: 'insert',
-                            containerPath: targetFolder,
-                            schemaName: 'study',
-                            queryName: 'Demographics',
-                            rows: [{
-                                Id: effectiveId,
-                                date: effectiveDate,
-                                alternateIds: row.Id !== effectiveId ? row.Id : null,
-                                gender: row.gender,
-                                species: row.species,
-                                birth: row.birth,
-                                death: row.death,
-                                dam: row.dam,
-                                sire: row.sire,
-                                damMccAlias: row['damMccAlias/externalAlias'],
-                                sireMccAlias: row['sireMccAlias/externalAlias'],
-                                colony: centerName,
-                                source: row.colony,
-                                calculated_status: 'Alive',
-                                mccAlias: row['Id/mccAlias/externalAlias'],
-                                QCState: null,
-                                QCStateLabel: 'Completed',
-                                objectId: null
-                            }]
-                        });
-
-                        commands.push({
-                            command: 'update',
-                            containerPath: null, //Use current folder
-                            schemaName: 'study',
-                            queryName: 'Demographics',
-                            rows: [{
-                                Id: row.Id, // NOTE: always change the original record
-                                excludeFromCensus: true
-                            }]
-                        });
-                    }
-                    else {
-                        // Otherwise update the existing:
-                        commands.push({
-                            command: 'update',
-                            containerPath: targetFolder,
-                            schemaName: 'study',
-                            queryName: 'Demographics',
-                            rows: [{
-                                Id: row.Id,
-                                date: effectiveDate,
-                                alternateIds: null,
-                                gender: row.gender,
-                                species: row.species,
-                                birth: row.birth,
-                                death: row.death,
-                                dam: row.dam,
-                                sire: row.sire,
-                                colony: centerName,
-                                source: row.colony,
-                                calculated_status: 'Alive',
-                                QCState: null,
-                                QCStateLabel: 'Completed',
-                                objectId: null
-                            }]
-                        });
+                LABKEY.Query.SelectRows({
+                    schemaName: 'study',
+                    queryName: 'Demographics',
+                    containerPath: targetFolder,
+                    filterArray: [LABKEY.Filter.create('Id', uniqueIds.join(';'), LABKEY.Filter.Types.IN)],
+                    columns: 'Id,gender,species,birth,death,dam,sire,damMccAlias/externalAlias,sireMccAlias/externalAlias,calculated_status,Id/mccAlias/externalAlias,colony,source,lsid,objectid',
+                    scope: this,
+                    failure: LDK.Utils.getErrorCallback(),
+                    success: function(existingIdResults) {
+                        var preexistingIdsInTargetFolder = {};
+                        Ext4.Array.forEach(existingIdResults.rows, function(r){
+                            preexistingIdsInTargetFolder[r.Id] = r;
+                        }, this);
+
+                        this.doSave(win, results, preexistingIdsInTargetFolder);
                     }
+                });
+            }
+        });
+    },
+
+    doSave: function(win, results, preexistingIdsInTargetFolder){
+        var effectiveDate = win.down('#effectiveDate').getValue();
+        var centerName = win.down('#centerName').getValue();
+        var targetFolder = win.down('#targetFolder').getValue();
+        var targetFolderId = win.down('#targetFolder').store.findRecord('Path', targetFolder).get('EntityId');
+
+        var commands = [];
+        var hadError = false;
+        Ext4.Array.forEach(results.rows, function(row){
+            var effectiveId = win.down('#usePreviousId-' + row.Id).getValue() ? row.Id : win.down('#newId-' + row.Id).getValue();
+            var requestId = win.down('#requestId-' + row.Id).getValue();
+            // This should be checked above, although perhaps case sensitivity could get involved:
+            LDK.Assert.assertNotEmpty('Missing effective ID after query', effectiveId);
+
+            var shouldAddDeparture = !row['Id/MostRecentDeparture/MostRecentDeparture'] ||
+                    row['Id/MostRecentDeparture/MostRecentDeparture'] !== Ext4.Date.format(row.effectiveDate, 'Y-m-d') ||
+                    row['Id/MostRecentDeparture/mccRequestId'] !== requestId ||
+                    row.Id !== effectiveId;
+            if (shouldAddDeparture) {
+                commands.push({
+                    command: 'insert',
+                    schemaName: 'study',
+                    queryName: 'Departure',
+                    rows: [{
+                        Id: row.Id,
+                        date: effectiveDate,
+                        source: row.colony,
+                        destination: centerName,
+                        mccRequestId: requestId,
+                        description: row.colony ? 'Original center: ' + row.colony : null,
+                        qcstate: null,
+                        objectId: null,
+                        QCStateLabel: 'Completed'
+                    }]
+                });
+            }
 
-                    var shouldAddArrival = !row['Id/MostRecentArrival/MostRecentArrival'] ||
-                            row['Id/MostRecentArrival/MostRecentArrival'] !== Ext4.Date.format(row.effectiveDate, 'Y-m-d') ||
-                            row['Id/MostRecentArrival/mccRequestId'] !== requestId ||
-                            row.Id !== effectiveId;
-                    if (shouldAddArrival) {
-                        // And also add an arrival record. NOTE: set the date after the departure to get status to update properly
-                        var arrivalDate = new Date(effectiveDate).setMinutes(effectiveDate.getMinutes() + 1);
-                        commands.push({
-                            command: 'insert',
-                            containerPath: targetFolder,
-                            schemaName: 'study',
-                            queryName: 'Arrival',
-                            rows: [{
-                                Id: effectiveId,
-                                date: arrivalDate,
-                                source: centerName,
-                                mccRequestId: requestId,
-                                QCState: null,
-                                QCStateLabel: 'Completed',
-                                objectId: null
-                            }]
-                        });
+            // If going to a new LK folder, we're creating a whole new record:
+            if (targetFolderId.toUpperCase() !== LABKEY.Security.currentContainer.id.toUpperCase() || effectiveId !== row.Id) {
+                if (Ext4.Object.getKeys(preexistingIdsInTargetFolder).indexOf(effectiveId) === -1) {
+                    // No existing record for this ID, make new record:
+                    commands.push({
+                        command: 'insert',
+                        containerPath: targetFolder,
+                        schemaName: 'study',
+                        queryName: 'Demographics',
+                        rows: [{
+                            Id: effectiveId,
+                            date: effectiveDate,
+                            alternateIds: row.Id !== effectiveId ? row.Id : null,
+                            gender: row.gender,
+                            species: row.species,
+                            birth: row.birth,
+                            death: row.death,
+                            dam: row.dam,
+                            sire: row.sire,
+                            damMccAlias: row['damMccAlias/externalAlias'],
+                            sireMccAlias: row['sireMccAlias/externalAlias'],
+                            colony: centerName,
+                            source: row.colony,
+                            calculated_status: 'Alive',
+                            mccAlias: row['Id/mccAlias/externalAlias'],
+                            QCState: null,
+                            QCStateLabel: 'Completed',
+                            objectId: null
+                        }]
+                    });
+                }
+                else {
+                    // There is an existing record for this ID, so merge/validate:
+                    console.log('Existing record found for: ' + effectiveId)
+                    var toUpdate = preexistingIdsInTargetFolder[effectiveId]
+
+                    var errors = []
+                    Ext4.Array.forEach(['gender', 'species', 'birth', 'death', 'dam', 'sire'], function(fieldName) {
+                        this.doFieldCheck(row, fieldName, toUpdate, fieldName, errors, effectiveId)
+                    }, this);
+
+                    toUpdate.colony = centerName
+                    toUpdate.source = toUpdate.source || row.colony
+                    toUpdate.calculated_status = toUpdate.calculated_status || 'Alive';
+
+                    if (row.Id !== effectiveId) {
+                        toUpdate.alternateIds = toUpdate.alternateIds ? toUpdate.alternateIds + ',' + row.Id : row.Id;
                     }
-                }, this);
 
-                LABKEY.Query.saveRows({
-                    commands: commands,
-                    scope: this,
-                    failure: LDK.Utils.getErrorCallback(),
-                    success: function() {
-                        Ext4.Msg.hide();
-                        Ext4.Msg.alert('Success', 'Transfer Added', function () {
-                            var dataRegion = LABKEY.DataRegions[this.dataRegionName];
-                            this.destroy();
+                    this.doFieldCheck(row, 'damMccAlias/externalAlias', toUpdate, 'damMccAlias', errors, effectiveId)
+                    this.doFieldCheck(row, 'sireMccAlias/externalAlias', toUpdate, 'sireMccAlias', errors, effectiveId)
+                    this.doFieldCheck(row, 'Id/mccAlias/externalAlias', toUpdate, 'mccAlias', errors, effectiveId)
 
-                            dataRegion.refresh();
-                        }, this);
+                    if (errors.length) {
+                        Ext4.Msg.hide();
+                        Ext4.Msg.alert('Error', 'Inconsistent data between source and destination demographics for: ' + effectiveId + + '<br>' + errors.join('<br>'));
+                        hadError = true;
+                        return false;
                     }
+
+                    commands.push({
+                        command: 'update',
+                        containerPath: targetFolder,
+                        schemaName: 'study',
+                        queryName: 'Demographics',
+                        rows: [toUpdate]
+                    });
+                }
+
+                commands.push({
+                    command: 'update',
+                    containerPath: null, //Use current folder
+                    schemaName: 'study',
+                    queryName: 'Demographics',
+                    rows: [{
+                        Id: row.Id, // NOTE: always change the original record
+                        excludeFromCensus: true
+                    }]
                 });
             }
+            else {
+                // Otherwise update the existing:
+                commands.push({
+                    command: 'update',
+                    containerPath: targetFolder,
+                    schemaName: 'study',
+                    queryName: 'Demographics',
+                    rows: [{
+                        Id: row.Id,
+                        date: effectiveDate,
+                        alternateIds: null,
+                        gender: row.gender,
+                        species: row.species,
+                        birth: row.birth,
+                        death: row.death,
+                        dam: row.dam,
+                        sire: row.sire,
+                        colony: centerName,
+                        source: row.colony,
+                        calculated_status: 'Alive',
+                        QCState: null,
+                        QCStateLabel: 'Completed',
+                        objectId: null
+                    }]
+                });
+            }
+
+            var shouldAddArrival = !row['Id/MostRecentArrival/MostRecentArrival'] ||
+                    row['Id/MostRecentArrival/MostRecentArrival'] !== Ext4.Date.format(row.effectiveDate, 'Y-m-d') ||
+                    row['Id/MostRecentArrival/mccRequestId'] !== requestId ||
+                    row.Id !== effectiveId;
+            if (shouldAddArrival) {
+                // And also add an arrival record. NOTE: set the date after the departure to get status to update properly
+                var arrivalDate = new Date(effectiveDate).setMinutes(effectiveDate.getMinutes() + 1);
+                commands.push({
+                    command: 'insert',
+                    containerPath: targetFolder,
+                    schemaName: 'study',
+                    queryName: 'Arrival',
+                    rows: [{
+                        Id: effectiveId,
+                        date: arrivalDate,
+                        source: centerName,
+                        mccRequestId: requestId,
+                        QCState: null,
+                        QCStateLabel: 'Completed',
+                        objectId: null
+                    }]
+                });
+            }
+        }, this);
+
+        if (hadError) {
+            return;
+        }
+
+        LABKEY.Query.saveRows({
+            commands: commands,
+            scope: this,
+            failure: LDK.Utils.getErrorCallback(),
+            success: function() {
+                Ext4.Msg.hide();
+                Ext4.Msg.alert('Success', 'Transfer Added', function () {
+                    var dataRegion = LABKEY.DataRegions[this.dataRegionName];
+                    this.destroy();
+
+                    dataRegion.refresh();
+                }, this);
+            }
         });
+    },
+
+    doFieldCheck: function(row, fieldName1, toUpdate, fieldName2, errors, effectiveId) {
+        if (row[fieldName1]) {
+            if (toUpdate[fieldName2] && toUpdate[fieldName2] !== row[fieldName1]) {
+                errors.push('Pre-existing record for ' + effectiveId + ', but ' + fieldName2 + ' was inconsistent between old/new (' + toUpdate[fieldName2] + '/' + row[fieldName1] + ')')
+            }
+            else {
+                toUpdate[fieldName2] = row[fieldName1];
+            }
+        }
     }
 });
\ No newline at end of file

From 0496cb387ea1475db13f731a0db5314a2d8d7173 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sun, 2 Nov 2025 06:30:35 -0800
Subject: [PATCH 39/40] Bugfix to MarkShippedWindow

---
 mcc/resources/web/mcc/window/MarkShippedWindow.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mcc/resources/web/mcc/window/MarkShippedWindow.js b/mcc/resources/web/mcc/window/MarkShippedWindow.js
index a38fc3abc..9cbf6818e 100644
--- a/mcc/resources/web/mcc/window/MarkShippedWindow.js
+++ b/mcc/resources/web/mcc/window/MarkShippedWindow.js
@@ -228,7 +228,7 @@ Ext4.define('MCC.window.MarkShippedWindow', {
                     uniqueIds.push(win.down('#usePreviousId-' + row.Id).getValue() ? row.Id : win.down('#newId-' + row.Id).getValue());
                 }, this);
 
-                LABKEY.Query.SelectRows({
+                LABKEY.Query.selectRows({
                     schemaName: 'study',
                     queryName: 'Demographics',
                     containerPath: targetFolder,

From 16681ba801bdba513bfdc8c504aaeb3cc473eba2 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 5 Nov 2025 07:14:26 -0800
Subject: [PATCH 40/40] Auto-create demographics records from SIV study
 assignment

---
 .../SivStudiesDataValidationNotification.java |  35 ++++-
 .../query/AutoCreateDemographicsTrigger.java  | 127 ++++++++++++++++++
 .../query/DefaultDatasetTrigger.java          |  29 ++++
 .../query/SivStudiesCustomizer.java           |   1 +
 mcc/resources/etls/mcc.xml                    |   2 +-
 5 files changed, 192 insertions(+), 2 deletions(-)
 create mode 100644 SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java

diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
index faeae93c4..24c03dbbf 100644
--- a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
+++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
@@ -1,20 +1,30 @@
 package org.labkey.sivstudies.notification;
 
+import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.CompareType;
 import org.labkey.api.data.Container;
+import org.labkey.api.data.SimpleFilter;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.ldk.notification.AbstractNotification;
 import org.labkey.api.module.ModuleLoader;
+import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.QueryService;
 import org.labkey.api.query.UserSchema;
 import org.labkey.api.security.User;
+import org.labkey.api.study.Study;
+import org.labkey.api.study.StudyService;
+import org.labkey.api.util.logging.LogHelper;
 import org.labkey.sivstudies.SivStudiesModule;
+import org.labkey.sivstudies.query.DefaultDatasetTrigger;
 
 import java.util.Date;
 
 public class SivStudiesDataValidationNotification extends AbstractNotification
 {
+    protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to SivStudiesDataValidationNotification");
+
     public SivStudiesDataValidationNotification()
     {
         super(ModuleLoader.getInstance().getModule(SivStudiesModule.class));
@@ -65,6 +75,7 @@ public String getEmailSubject(Container c)
         duplicateInfectionCheck(c, u, msg);
         infectionAnchorDateDiscordance(c, u, msg);
         pvlWithoutInfectionDate(c, u, msg);
+        idsMissingFromDemographics(c, u, msg);
 
         if (!msg.isEmpty())
         {
@@ -102,10 +113,15 @@ private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder m
     }
 
     private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message)
+    {
+        genericQueryCheck(c, u, msg, schemaName, queryName, message, null);
+    }
+
+    private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message, @Nullable SimpleFilter filter)
     {
         TableInfo ti = getTableInfo(u, c, schemaName, queryName);
 
-        TableSelector ts = new TableSelector(ti);
+        TableSelector ts = new TableSelector(ti, filter, null);
         long count = ts.getRowCount();
         if (count > 0)
         {
@@ -119,4 +135,21 @@ private void pvlWithoutInfectionDate(Container c, User u, StringBuilder msg)
     {
         genericQueryCheck(c, u, msg, "study", "pvlWithoutInfectionDate", "animals with PVL data but no record of SIV infection");
     }
+
+    private void idsMissingFromDemographics(Container c, User u, StringBuilder msg)
+    {
+        Study s = StudyService.get().getStudy(getTargetContainer(c));
+        if (s == null)
+        {
+            return;
+        }
+
+        SimpleFilter filter = new SimpleFilter(FieldKey.fromString("DataSet/Demographics/" + s.getSubjectColumnName()), null, CompareType.ISBLANK);
+        genericQueryCheck(c, u, msg, "study", s.getSubjectNounSingular(), "IDs with data in the study not present in the demographics table", filter);
+    }
+
+    protected Container getTargetContainer(Container c)
+    {
+        return c.isWorkbookOrTab() ? c.getParent() : c;
+    }
 }
diff --git a/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java
new file mode 100644
index 000000000..00e5b7394
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java
@@ -0,0 +1,127 @@
+package org.labkey.sivstudies.query;
+
+import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.collections.CaseInsensitiveHashSet;
+import org.labkey.api.data.CompareType;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.SimpleFilter;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.data.triggers.Trigger;
+import org.labkey.api.data.triggers.TriggerFactory;
+import org.labkey.api.query.BatchValidationException;
+import org.labkey.api.query.DuplicateKeyException;
+import org.labkey.api.query.FieldKey;
+import org.labkey.api.query.QueryService;
+import org.labkey.api.query.QueryUpdateServiceException;
+import org.labkey.api.query.ValidationException;
+import org.labkey.api.security.User;
+import org.labkey.api.study.Study;
+import org.labkey.api.study.StudyService;
+import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.util.logging.LogHelper;
+
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class AutoCreateDemographicsTrigger extends DefaultDatasetTrigger
+{
+    protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to CreateDemographicsTrigger");
+
+    public static class Factory implements TriggerFactory
+    {
+        public Factory()
+        {
+
+        }
+
+        @Override
+        public @NotNull Collection<Trigger> createTrigger(@Nullable Container c, TableInfo table, Map<String, Object> extraContext)
+        {
+            return List.of(new AutoCreateDemographicsTrigger());
+        }
+    }
+
+    private static final String CACHE_KEY = "~~AutoCreateDemographicsTrigger.IdsToCreate~~";
+
+    @Override
+    protected void afterUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        if (extraContext == null)
+        {
+            _log.error("extraContext is null in AutoCreateDemographicsTrigger.afterUpsert()");
+            return;
+        }
+
+        if (!extraContext.containsKey(AutoCreateDemographicsTrigger.CACHE_KEY))
+        {
+            extraContext.put(CACHE_KEY, new CaseInsensitiveHashSet());
+        }
+
+        if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s)
+        {
+            String idField = getIdField(c);
+            String id = newRow.get(idField) != null  ? newRow.get(idField).toString() : null;
+            if (id != null)
+            {
+                s.add(id);
+            }
+        }
+    }
+
+    @Override
+    public void complete(TableInfo table, Container c, User user, TableInfo.TriggerType event, BatchValidationException errors, Map<String, Object> extraContext)
+    {
+        if (extraContext == null)
+        {
+            _log.error("extraContext is null in AutoCreateDemographicsTrigger.complete()");
+            return;
+        }
+
+        if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s)
+        {
+            s = new CaseInsensitiveHashSet(s);
+
+            String idField = getIdField(c);
+            TableInfo ti = QueryService.get().getUserSchema(user, getTargetContainer(c), "study").getTable("demographics");
+            List<String> existingIds = new TableSelector(ti, PageFlowUtil.set(idField), new SimpleFilter(FieldKey.fromString(idField), s, CompareType.IN), null).getArrayList(String.class);
+
+            s.removeAll(existingIds);
+
+            if (!s.isEmpty())
+            {
+                List<Map<String, Object>> toInsert = s.stream().map(id -> Map.of(idField, (Object)id)).toList();
+                try
+                {
+                    ti.getUpdateService().insertRows(user, c, toInsert, null, null, null);
+                }
+                catch (SQLException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e)
+                {
+                    _log.error("Error creating demographics records", e);
+                }
+            }
+        }
+    }
+
+    private String _idField = null;
+
+    private String getIdField(Container c)
+    {
+        if (_idField == null)
+        {
+            Study s = StudyService.get().getStudy(getTargetContainer(c));
+            if (s == null)
+            {
+                return null;
+            }
+
+            _idField = s.getSubjectColumnName();
+        }
+
+        return _idField;
+    }
+}
diff --git a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
index 26efb1912..fea3c8899 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
@@ -40,6 +40,29 @@ public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<
         beforeInsert(table, c, user, newRow, errors, extraContext, null);
     }
 
+    @Override
+    public void afterInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext, @Nullable Map<String, Object> existingRecord) throws ValidationException
+    {
+        afterUpsert(table, c, user, newRow, existingRecord, errors, extraContext);
+    }
+
+    @Override
+    public void afterInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        afterInsert(table, c, user, newRow, errors, extraContext, null);
+    }
+
+    @Override
+    public void afterUpdate(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        afterUpsert(table, c, user, newRow, oldRow, errors, extraContext);
+    }
+
+    protected void afterUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+
+    }
+
     @Override
     public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext, @Nullable Map<String, Object> existingRecord) throws ValidationException
     {
@@ -84,4 +107,10 @@ private void mergeOldToNewRow(@NotNull Map<String, Object> newRow, @Nullable Map
             }
         }
     }
+
+    protected Container getTargetContainer(Container c)
+    {
+        return c.isWorkbookOrTab() ? c.getParent() : c;
+    }
+
 }
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 4a8f3e36d..1fc15ddbb 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -79,6 +79,7 @@ public void performDatasetCustomization(DatasetTable ds)
             if ("assignment".equalsIgnoreCase(ds.getName()))
             {
                 ati.addTriggerFactory(StudiesService.get().getStudiesTriggerFactory());
+                ati.addTriggerFactory(new AutoCreateDemographicsTrigger.Factory());
             }
         }
         else
diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml
index d324ce40e..e5a788736 100644
--- a/mcc/resources/etls/mcc.xml
+++ b/mcc/resources/etls/mcc.xml
@@ -38,7 +38,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true" batchSize="5000">
+            <destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true" batchSize="2500">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>