The fields to calculate timePostSivChallenge require a record in studies.subjectAnchorDates from the same Id where eventLabel = 'SIV Infection'. There may be records in the treatments table for SIV challenges; these do not count
+
The fields to calculate overlapping PVLs require a record in the viral_load table from the same Id/date, where target = SIV and where sampleType = plasma.
+
The fields to calculate ART-related date require record(s) in the treatments table from the same Id, where category = 'ART'
+
\ No newline at end of file
diff --git a/SivStudies/resources/views/studiesAdmin.html b/SivStudies/resources/views/studiesAdmin.html
index fb5e91025..8cebaf45b 100644
--- a/SivStudies/resources/views/studiesAdmin.html
+++ b/SivStudies/resources/views/studiesAdmin.html
@@ -27,6 +27,9 @@
},{
name: 'Notification Admin',
url: LABKEY.ActionURL.buildURL('ldk', 'notificationAdmin.view')
+ },{
+ name: 'Notes For Managing Data',
+ url: LABKEY.ActionURL.buildURL('sivstudies', 'dataNotes.view')
}]
}]
}]
diff --git a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
new file mode 100644
index 000000000..fb8f9de4c
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
@@ -0,0 +1,100 @@
+package org.labkey.sivstudies.etl;
+
+import org.apache.xmlbeans.XmlException;
+import org.jetbrains.annotations.NotNull;
+import org.labkey.api.collections.CaseInsensitiveHashSet;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.di.TaskRefTask;
+import org.labkey.api.pipeline.PipelineJob;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.pipeline.RecordedActionSet;
+import org.labkey.api.query.BatchValidationException;
+import org.labkey.api.query.DuplicateKeyException;
+import org.labkey.api.query.QueryService;
+import org.labkey.api.query.QueryUpdateServiceException;
+import org.labkey.api.query.UserSchema;
+import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.writer.ContainerUser;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class AddMissingIdrSubjects implements TaskRefTask
+{
+ protected ContainerUser _containerUser;
+
+ @Override
+ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJobException
+ {
+ // Find existing IDs:
+ UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study");
+ if (us == null)
+ {
+ throw new PipelineJobException("Missing study schema");
+ }
+
+ List existingIds = new ArrayList<>(new TableSelector(us.getTable("demographics"), PageFlowUtil.set("Id"), null, null).getArrayList(String.class));
+
+ // Source IDs:
+ Container sourceContainer = ContainerManager.getForPath("Labs/Bimber");
+ UserSchema us2 = QueryService.get().getUserSchema(_containerUser.getUser(), sourceContainer, "bimber_data");
+ if (us2 == null)
+ {
+ throw new PipelineJobException("Missing bimber_data schema");
+ }
+
+ List allIds = new ArrayList<>(new TableSelector(us2.getTable("subjects"), PageFlowUtil.set("Rh"), null, null).getArrayList(String.class));
+
+ allIds.removeAll(existingIds);
+ if (allIds.isEmpty())
+ {
+ return new RecordedActionSet();
+ }
+
+ allIds = new ArrayList<>(new CaseInsensitiveHashSet(allIds));
+
+ pipelineJob.getLogger().info("Creating {} subjects", allIds.size());
+ List> toInsert = new ArrayList<>();
+ allIds.forEach(id -> {
+ toInsert.add(Map.of("Id", id));
+ });
+
+ try
+ {
+ BatchValidationException bve = new BatchValidationException();
+ us.getTable("demographics").getUpdateService().insertRows(_containerUser.getUser(), _containerUser.getContainer(), toInsert, bve, null, null);
+ if (bve.hasErrors())
+ {
+ throw bve;
+ }
+ }
+ catch (BatchValidationException | SQLException | DuplicateKeyException | QueryUpdateServiceException e)
+ {
+ throw new PipelineJobException(e);
+ }
+
+ return new RecordedActionSet();
+ }
+
+ @Override
+ public List getRequiredSettings()
+ {
+ return List.of();
+ }
+
+ @Override
+ public void setSettings(Map map) throws XmlException
+ {
+
+ }
+
+ @Override
+ public void setContainerUser(ContainerUser containerUser)
+ {
+ _containerUser = containerUser;
+ }
+}
diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
index 6c750b8bf..37b54fa8d 100644
--- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
+++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
@@ -101,7 +101,7 @@ public boolean isRequired()
}
}
- final int BATCH_SIZE = 250;
+ final int BATCH_SIZE = 500;
private MODE getMode()
{
diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
index 314c662a0..24c03dbbf 100644
--- a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
+++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
@@ -1,20 +1,30 @@
package org.labkey.sivstudies.notification;
+import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.CompareType;
import org.labkey.api.data.Container;
+import org.labkey.api.data.SimpleFilter;
import org.labkey.api.data.TableInfo;
import org.labkey.api.data.TableSelector;
import org.labkey.api.ldk.notification.AbstractNotification;
import org.labkey.api.module.ModuleLoader;
+import org.labkey.api.query.FieldKey;
import org.labkey.api.query.QueryService;
import org.labkey.api.query.UserSchema;
import org.labkey.api.security.User;
+import org.labkey.api.study.Study;
+import org.labkey.api.study.StudyService;
+import org.labkey.api.util.logging.LogHelper;
import org.labkey.sivstudies.SivStudiesModule;
+import org.labkey.sivstudies.query.DefaultDatasetTrigger;
import java.util.Date;
public class SivStudiesDataValidationNotification extends AbstractNotification
{
+ protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to SivStudiesDataValidationNotification");
+
public SivStudiesDataValidationNotification()
{
super(ModuleLoader.getInstance().getModule(SivStudiesModule.class));
@@ -63,6 +73,9 @@ public String getEmailSubject(Container c)
Date now = new Date();
duplicateInfectionCheck(c, u, msg);
+ infectionAnchorDateDiscordance(c, u, msg);
+ pvlWithoutInfectionDate(c, u, msg);
+ idsMissingFromDemographics(c, u, msg);
if (!msg.isEmpty())
{
@@ -91,35 +104,52 @@ private TableInfo getTableInfo(User u, Container c, String schemaName, String qu
private void duplicateInfectionCheck(Container c, User u, StringBuilder msg)
{
- String schemaName = "study";
- String queryName = "duplicateInfectionDates";
+ genericQueryCheck(c, u, msg, "study", "duplicateInfectionDates", "duplicate infection date records");
+ }
+
+ private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder msg)
+ {
+ genericQueryCheck(c, u, msg, "study", "infectionAnchorDateDiscordance", "records with discordant treatment and anchor date SIV infection records");
+ }
+
+ private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message)
+ {
+ genericQueryCheck(c, u, msg, schemaName, queryName, message, null);
+ }
+ private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message, @Nullable SimpleFilter filter)
+ {
TableInfo ti = getTableInfo(u, c, schemaName, queryName);
- TableSelector ts = new TableSelector(ti);
+ TableSelector ts = new TableSelector(ti, filter, null);
long count = ts.getRowCount();
if (count > 0)
{
- msg.append("WARNING: There are ").append(count).append(" duplicate infection date records \n");
+ msg.append("WARNING: There are ").append(count).append(" " + message + " \n");
msg.append("
\n\n");
}
}
- private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder msg)
+ private void pvlWithoutInfectionDate(Container c, User u, StringBuilder msg)
{
- String schemaName = "study";
- String queryName = "infectionAnchorDateDiscordance";
-
- TableInfo ti = getTableInfo(u, c, schemaName, queryName);
+ genericQueryCheck(c, u, msg, "study", "pvlWithoutInfectionDate", "animals with PVL data but no record of SIV infection");
+ }
- TableSelector ts = new TableSelector(ti);
- long count = ts.getRowCount();
- if (count > 0)
+ private void idsMissingFromDemographics(Container c, User u, StringBuilder msg)
+ {
+ Study s = StudyService.get().getStudy(getTargetContainer(c));
+ if (s == null)
{
- msg.append("WARNING: There are ").append(count).append(" records with discordant treatment and anchor date SIV infection records \n");
- msg.append("