Merge discvr-25.3 to develop

SivStudies/resources/queries/study/additionalDatatypes.query.xml

@@ -4,7 +4,7 @@
             <table tableName="additionalDatatypes" tableDbType="TABLE">
                 <columns>
                     <column columnName="Id">
-                        <url />
+
                     </column>
                     <column columnName="date">
 

SivStudies/resources/queries/study/demographics.query.xml

@@ -5,7 +5,6 @@
                 <javaCustomizer class="org.labkey.sivstudies.query.SivStudiesCustomizer"/>
                 <columns>
                     <column columnName="Id">
-                        <url />
                         <isKeyField>true</isKeyField>
                     </column>
                     <column columnName="date">

SivStudies/resources/queries/study/demographics/Expanded.qview.xml

@@ -9,6 +9,8 @@
         <column name="projects/allStudies"/>
         <column name="projects/categories"/>
         <column name="immunizations/immunizations"/>
+        <column name="sivART/allInfections"/>
+        <column name="sivART/allART"/>
         <column name="outcomes/outcomes"/>
     </columns>
     <sorts>

SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml

@@ -0,0 +1,22 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="" tableDbType="NOT_IN_DB">
+                <tableTitle>SIV/ART Summary</tableTitle>
+                <columns>
+                    <column columnName="Id">
+                        <isKeyField>true</isKeyField>
+                        <isHidden>true</isHidden>
+                    </column>
+                    <column columnName="allInfections">
+                        <columnTitle>SIV Infection</columnTitle>
+                    </column>
+                    <column columnName="allART">
+                        <columnTitle>ART</columnTitle>
+                    </column>
+                </columns>
+                <titleColumn>allInfections</titleColumn>
+            </table>
+        </tables>
+    </metadata>
+</query>

SivStudies/resources/queries/study/demographicsChallengeAndArt.sql

@@ -0,0 +1,13 @@
+SELECT
+  t.Id,
+  group_concat(DISTINCT CASE
+      WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')')
+      ELSE NULL
+  END, char(10)) as allInfections,
+  group_concat(DISTINCT CASE
+      WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')')
+      ELSE NULL
+  END, char(10)) as allART,
+
+FROM study.treatments t
+GROUP BY t.Id

SivStudies/resources/queries/study/procedures.query.xml

@@ -4,7 +4,7 @@
             <table tableName="procedures" tableDbType="TABLE">
                 <columns>
                     <column columnName="Id">
-                        <url />
+
                     </column>
                     <column columnName="date">
 

SivStudies/resources/queries/study/studyData.query.xml

@@ -6,6 +6,7 @@
                 <columns>
                     <column columnName="Id">
                         <conceptURI>http://cpas.labkey.com/Study#ParticipantId</conceptURI>
+                        <url>laboratory/dataBrowser.view?subjectId=${Id}</url>
                     </column>
                     <column columnName="date">
                         <columnTitle>Date</columnTitle>
@@ -25,6 +26,9 @@
                     <column columnName="AnimalVisit">
                         <isHidden>true</isHidden>
                     </column>
+                    <column columnName="DataSets">
+                        <isHidden>true</isHidden>
+                    </column>
                 </columns>
                 <buttonBarOptions includeStandardButtons="true">
                     <item hidden="true">

SivStudies/resources/queries/study/weight.query.xml

@@ -4,7 +4,7 @@
             <table tableName="weight" tableDbType="TABLE">
                 <columns>
                     <column columnName="Id">
-                        <url />
+
                     </column>
                     <column columnName="date">
                         <isHidden>true</isHidden>

SivStudies/resources/views/participantView.html

@@ -0,0 +1,7 @@
+<script type="text/javascript" nonce="<%=scriptNonce%>">
+    // NOTE: this is intended to be set as the study participant view through the UI.
+    // Loading ClientDependencies is a pain through that mechanism, so this just redirects
+    Ext4.onReady(function (){
+        window.location = LABKEY.ActionURL.buildURL('laboratory', 'dataBrowser', null, {subjectId: LABKEY.ActionURL.getParameter('participantId')});
+    });
+</script>

SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java

@@ -9,6 +9,7 @@
 import org.labkey.api.data.CompareType;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.DbScope;
 import org.labkey.api.data.SimpleFilter;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
@@ -100,7 +101,7 @@ public boolean isRequired()
         }
     }
 
-    final int BATCH_SIZE = 100;
+    final int BATCH_SIZE = 250;
 
     private MODE getMode()
     {
@@ -136,127 +137,136 @@ private void checkCancelled(PipelineJob job)
     private void processBatch(List<String> subjects, Logger log, PipelineJob job)
     {
         log.info("processing batch with " + subjects.size() + " subjects");
-        TableInfo destinationTable = getDataDestinationTable();
+        try (DbScope.Transaction t = DbScope.getLabKeyScope().ensureTransaction())
+        {
+            TableInfo destinationTable = getDataDestinationTable();
 
-        QueryUpdateService qus = destinationTable.getUpdateService();
-        qus.setBulkLoad(true);
+            QueryUpdateService qus = destinationTable.getUpdateService();
+            qus.setBulkLoad(true);
 
-        try
-        {
-            if (getMode() == MODE.TRUNCATE)
+            try
             {
-                // Find / Delete existing values:
-                Set<ColumnInfo> keyFields = destinationTable.getColumns().stream().filter(ColumnInfo::isKeyField).collect(Collectors.toSet());
-                final SimpleFilter subjectFilter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name())), subjects, CompareType.IN);
-                if (_settings.get(Settings.targetAdditionalFilters.name()) != null)
+                if (getMode() == MODE.TRUNCATE)
                 {
-                    List<CompareType.AbstractCompareClause> additionalFilters = parseAdditionalFilters(_settings.get(Settings.targetAdditionalFilters.name()));
-                    additionalFilters.forEach(subjectFilter::addCondition);
-                }
+                    // Find / Delete existing values:
+                    Set<ColumnInfo> keyFields = destinationTable.getColumns().stream().filter(ColumnInfo::isKeyField).collect(Collectors.toSet());
+                    final SimpleFilter subjectFilter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name())), subjects, CompareType.IN);
+                    if (_settings.get(Settings.targetAdditionalFilters.name()) != null)
+                    {
+                        List<CompareType.AbstractCompareClause> additionalFilters = parseAdditionalFilters(_settings.get(Settings.targetAdditionalFilters.name()));
+                        additionalFilters.forEach(subjectFilter::addCondition);
+                    }
 
-                if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null)
-                {
-                    throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name()));
-                }
+                    if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null)
+                    {
+                        throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name()));
+                    }
 
-                List<Map<String, Object>> existingRows = new ArrayList<>(new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection());
-                if (!existingRows.isEmpty())
-                {
-                    List<List<Map<String, Object>>> batches = Lists.partition(existingRows, 5000);
-                    log.info("deleting " + existingRows.size() + " rows in " + batches.size() + " batches");
-                    int i = 0;
-                    for (List<Map<String, Object>> batch : batches)
+                    List<Map<String, Object>> existingRows = new ArrayList<>(new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection());
+                    if (!existingRows.isEmpty())
                     {
-                        i++;
-                        log.info("batch " + i);
-                        checkCancelled(job);
+                        List<List<Map<String, Object>>> batches = Lists.partition(existingRows, 5000);
+                        log.info("deleting " + existingRows.size() + " rows in " + batches.size() + " batches");
+                        int i = 0;
+                        for (List<Map<String, Object>> batch : batches)
+                        {
+                            i++;
+                            log.info("batch " + i);
+                            checkCancelled(job);
 
-                        qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), batch, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null);
+                            qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), batch, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null);
+                            t.commitAndKeepConnection();
+                        }
+                    }
+                    else
+                    {
+                        log.info("No rows to delete for this subject batch");
                     }
                 }
                 else
                 {
-                    log.info("No rows to delete for this subject batch");
+                    log.info("Using " + getMode().name() + " mode, source records will not be deleted");
                 }
-            }
-            else
-            {
-                log.info("Using " + getMode().name() + " mode, source records will not be deleted");
-            }
 
-            // Query data and import
-            List<Map<String, Object>> toImportOrUpdate = getRowsToImport(subjects, log);
-            if (!toImportOrUpdate.isEmpty())
-            {
-                if (getMode() == MODE.TRUNCATE)
+                // Query data and import
+                List<Map<String, Object>> toImportOrUpdate = getRowsToImport(subjects, log);
+                if (!toImportOrUpdate.isEmpty())
                 {
-                    List<List<Map<String, Object>>> batches = Lists.partition(toImportOrUpdate, 5000);
-                    log.info("inserting " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches");
-
-                    int i = 0;
-                    for (List<Map<String, Object>> batch : batches)
+                    if (getMode() == MODE.TRUNCATE)
                     {
-                        i++;
-                        log.info("batch " + i);
-                        checkCancelled(job);
+                        List<List<Map<String, Object>>> batches = Lists.partition(toImportOrUpdate, 5000);
+                        log.info("inserting " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches");
 
-                        BatchValidationException bve = new BatchValidationException();
-                        qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), batch, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null);
-                        if (bve.hasErrors())
+                        int i = 0;
+                        for (List<Map<String, Object>> batch : batches)
                         {
-                            throw bve;
+                            i++;
+                            log.info("batch " + i);
+                            checkCancelled(job);
+
+                            BatchValidationException bve = new BatchValidationException();
+                            qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), batch, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null);
+                            if (bve.hasErrors())
+                            {
+                                throw bve;
+                            }
+                            t.commitAndKeepConnection();
                         }
                     }
-                }
-                else if (getMode() == MODE.UPDATE_ONLY)
-                {
-                    List<List<Map<String, Object>>> batches = Lists.partition(toImportOrUpdate, 5000);
-                    log.info("updating " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches");
-
-                    int i = 0;
-                    for (List<Map<String, Object>> batch : batches)
+                    else if (getMode() == MODE.UPDATE_ONLY)
                     {
+                        List<List<Map<String, Object>>> batches = Lists.partition(toImportOrUpdate, 5000);
+                        log.info("updating " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches");
 
-                        i++;
-                        log.info("batch " + i);
-                        checkCancelled(job);
+                        int i = 0;
+                        for (List<Map<String, Object>> batch : batches)
+                        {
 
-                        BatchValidationException bve = new BatchValidationException();
+                            i++;
+                            log.info("batch " + i);
+                            checkCancelled(job);
 
-                        Collection<String> keyFields = destinationTable.getPkColumnNames();
-                        List<Map<String, Object>> keys = batch.stream().map(x -> {
-                            Map<String, Object> map = new HashMap<>();
-                            for (String keyField : keyFields)
-                            {
-                                if (x.get(keyField) != null)
+                            BatchValidationException bve = new BatchValidationException();
+
+                            Collection<String> keyFields = destinationTable.getPkColumnNames();
+                            List<Map<String, Object>> keys = batch.stream().map(x -> {
+                                Map<String, Object> map = new HashMap<>();
+                                for (String keyField : keyFields)
                                 {
-                                    map.put(keyField, x.get(keyField));
+                                    if (x.get(keyField) != null)
+                                    {
+                                        map.put(keyField, x.get(keyField));
+                                    }
                                 }
-                            }
 
-                            return map;
-                        }).toList();
+                                return map;
+                            }).toList();
 
-                        qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), batch, keys, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null);
-                        if (bve.hasErrors())
-                        {
-                            throw bve;
+                            qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), batch, keys, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null);
+                            if (bve.hasErrors())
+                            {
+                                throw bve;
+                            }
+                            t.commitAndKeepConnection();
                         }
                     }
+                    else
+                    {
+                        throw new IllegalStateException("Unknown mode: " + getMode());
+                    }
                 }
                 else
                 {
-                    throw new IllegalStateException("Unknown mode: " + getMode());
+                    log.info("No rows to import/update for this subject batch");
                 }
             }
-            else
+            catch (SQLException | InvalidKeyException | BatchValidationException | QueryUpdateServiceException |
+                   DuplicateKeyException e)
             {
-                log.info("No rows to import/update for this subject batch");
+                throw new IllegalStateException("Error Importing/Updating Rows", e);
             }
-        }
-        catch (SQLException | InvalidKeyException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e)
-        {
-            throw new IllegalStateException("Error Importing/Updating Rows", e);
+
+            t.commit();
         }
     }
 

SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java

@@ -249,6 +249,13 @@ private void appendDemographicsColumns(AbstractTableInfo parentTable)
             colInfo.setLabel("Outcomes");
             parentTable.addColumn(colInfo);
         }
+
+        if (parentTable.getColumn("sivART") == null)
+        {
+            BaseColumnInfo colInfo = getWrappedIdCol(parentTable.getUserSchema(), "demographicsChallengeAndArt", parentTable, "sivART");
+            colInfo.setLabel("SIV/ART Dates");
+            parentTable.addColumn(colInfo);
+        }
     }
 
     private void appendPvlColumns(DatasetTable ds, String subjectColName, String dateColName)

mGAP/resources/views/login.html

@@ -14,7 +14,8 @@
 
 </script>
 <form class="auth-form" name="login" method="post">
-    <div class="auth-header">Sign In</div>
+    <div class="auth-header">Sign In / Register</div>
+    <div style="max-width: 1000px;padding-top: 10px;">While mGAP is a free NIH-sponsored resource, we require users register to help us track and report usage to our funders. This information is critical to demonstrate the value and impact of the resource. Please <a href="<%=contextPath%>/mgap/home/requestLogin.view?">request an account</a> if you are not already registered. Thank you for your understanding.</div>
     <div class="labkey-error" id="errors"></div>
     <div class="auth-form-body">
         <label for="email">Email</label>