From 5048d12edb39d0c3f214a64902f30e56620be449 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 6 May 2025 14:19:57 -0700 Subject: [PATCH 01/34] More improvements to mGAP ETL --- .../AbstractVariantTransform.java | 3 +- .../JBrowseSessionTransform.java | 64 +++++++++++++++---- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java index 04530b42..184f2104 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java @@ -133,7 +133,7 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S File localCopy = doFileCopy(f, subDir, name); if (localCopy == null) { - // TODO + getStatusLogger().warn("localCopy was null", new Exception()); } //first create the ExpData @@ -217,7 +217,6 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip File localCopy = new File(subdir, name == null || f.getName().startsWith("mGap.v") ? f.getName() : FileUtil.makeLegalName(name).replaceAll(" ", "_") + ".vcf.gz"); if (f.equals(localCopy)) { - getStatusLogger().debug("Attempting to copy file that is already a child of the target: " + f.getPath(), new Exception()); return localCopy; } diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java index e489595d..b18bc2ed 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java @@ -4,13 +4,12 @@ import org.json.JSONObject; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; import org.labkey.api.data.Results; -import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.jbrowse.JBrowseService; @@ -30,8 +29,11 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; /** * Created by bimber on 5/15/2017. @@ -66,14 +68,20 @@ protected Object doTransform(Object inputValue) Integer outputFileId = getOrCreateOutputFile(input, getInputValue("objectId"), null); if (outputFileId != null) { - //find database ID, if exists: - //determine if there is already a JSONfile for this outputfile + //find database ID, if exists, based on name: UserSchema us = getJbrowseUserSchema(); - SQLFragment sql = new SQLFragment("SELECT m." + us.getDbSchema().getScope().getSqlDialect().makeLegalIdentifier("database") + " FROM jbrowse.jsonfiles j JOIN jbrowse.database_members m ON (j.objectId = m.jsonfile) WHERE j.outputfile = ?", outputFileId); - String databaseId = new SqlSelector(us.getDbSchema().getScope(), sql).getObject(String.class); + TableSelector ts = new TableSelector(us.getTable("databases"), PageFlowUtil.set("objectid"), new SimpleFilter(FieldKey.fromString("name"), getDatabaseName()), null); + String databaseId = ts.getObject(String.class); if (databaseId != null) { - getStatusLogger().info("jbrowse database exists using the output file: " + outputFileId); + getStatusLogger().info("jbrowse database exists using name: " + getDatabaseName()); + + boolean hadChanges = addTracks(databaseId, releaseId); + if (hadChanges) + { + recreateSession(databaseId); + } + return databaseId; } else @@ -81,7 +89,7 @@ protected Object doTransform(Object inputValue) try { databaseId = new GUID().toString(); - getStatusLogger().info("creating jbrowse database: " + databaseId); + getStatusLogger().info("creating jbrowse database: " + databaseId + ", for output file: " + outputFileId); //create database TableInfo databases = getJbrowseUserSchema().getTable("databases"); @@ -140,7 +148,7 @@ private void recreateSession(final String databaseId) }, DbScope.CommitTaskOption.POSTCOMMIT); } - protected void addTracks(final String databaseId, String releaseId) + protected boolean addTracks(final String databaseId, String releaseId) { //then JSONfiles/database members List fks = Arrays.asList( @@ -166,18 +174,48 @@ protected void addTracks(final String databaseId, String releaseId) getStatusLogger().error("no track records found for release: " + releaseId); } + final AtomicBoolean hadChanges = new AtomicBoolean(false); + final Set jsonFiles = new HashSet<>(); ts.forEachResults(rs -> { try { getStatusLogger().info("possibly creating track for: " + rs.getString(FieldKey.fromString("trackName"))); String jsonFile = getOrCreateJsonFile(rs, "vcfId/dataid/DataFileUrl"); - getOrCreateDatabaseMember(databaseId, jsonFile); + jsonFiles.add(jsonFile); + boolean added = getOrCreateDatabaseMember(databaseId, jsonFile); + if (added) + { + hadChanges.set(true); + } } catch (Exception e) { getStatusLogger().error(e.getMessage(), e); } }); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("database"), databaseId); + filter.addCondition(FieldKey.fromString("jsonfile"), jsonFiles, CompareType.NOT_IN); + TableSelector ts2 = new TableSelector(getDatabaseMembers(), PageFlowUtil.set("rowid"), filter, null); + if (ts2.exists()) + { + List rowIds = ts2.getArrayList(Integer.class); + + getStatusLogger().info("Deleting " + rowIds.size() + " database_member rows for: " + databaseId); + List> toDelete = new ArrayList<>(); + rowIds.forEach(rowId -> toDelete.add(new CaseInsensitiveHashMap<>(Map.of("rowid", rowId)))); + try + { + getJbrowseUserSchema().getTable("database_members").getUpdateService().deleteRows(getContainerUser().getUser(), getContainerUser().getContainer(), toDelete, null, null); + hadChanges.set(true); + } + catch (InvalidKeyException | BatchValidationException | SQLException |QueryUpdateServiceException e) + { + getStatusLogger().error(e); + } + } + + return hadChanges.get(); } private void ensureLuceneData(String objectId) @@ -208,7 +246,7 @@ private void ensureLuceneData(String objectId) } } - protected void getOrCreateDatabaseMember(String databaseId, String jsonFileId) throws Exception + protected boolean getOrCreateDatabaseMember(String databaseId, String jsonFileId) throws Exception { SimpleFilter filter = new SimpleFilter(FieldKey.fromString("database"), databaseId); filter.addCondition(FieldKey.fromString("jsonfile"), jsonFileId); @@ -216,7 +254,7 @@ protected void getOrCreateDatabaseMember(String databaseId, String jsonFileId) t if (new TableSelector(getDatabaseMembers(), filter, null).exists()) { getStatusLogger().info("database member exists for: " + jsonFileId); - return; + return false; } TableInfo databaseMembers = getJbrowseUserSchema().getTable("database_members"); @@ -232,6 +270,8 @@ protected void getOrCreateDatabaseMember(String databaseId, String jsonFileId) t getStatusLogger().info("creating database member for: " + jsonFileId); databaseMembers.getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(row), new BatchValidationException(), null, new HashMap<>()); + + return true; } protected TableInfo getJsonFiles() From 1fb9bc82c050a382d7688a2befc9f39133397a55 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 6 May 2025 14:27:11 -0700 Subject: [PATCH 02/34] Missed with prior commit --- .../mgap/columnTransforms/JBrowseHumanSessionTransform.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java index 4199f61f..b90b4d20 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java @@ -54,17 +54,18 @@ protected String getGenomeIdField() } @Override - protected void addTracks(final String databaseId, String releaseId) + protected boolean addTracks(final String databaseId, String releaseId) { try { getStatusLogger().info("possibly creating track for: " + getDatabaseName()); String jsonFile = getOrCreateJsonFile(); - getOrCreateDatabaseMember(databaseId, jsonFile); + return getOrCreateDatabaseMember(databaseId, jsonFile); } catch(Exception e) { getStatusLogger().error(e.getMessage(), e); + return false; } } From f0fc5b8a91db80ff39a28684dd593585516052ba Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 7 May 2025 06:07:18 -0700 Subject: [PATCH 03/34] Bugfix to mGAP ETL --- .../JBrowseSessionTransform.java | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java index b18bc2ed..2ce9b755 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java @@ -10,6 +10,7 @@ import org.labkey.api.data.DbScope; import org.labkey.api.data.Results; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.jbrowse.JBrowseService; @@ -70,10 +71,10 @@ protected Object doTransform(Object inputValue) { //find database ID, if exists, based on name: UserSchema us = getJbrowseUserSchema(); - TableSelector ts = new TableSelector(us.getTable("databases"), PageFlowUtil.set("objectid"), new SimpleFilter(FieldKey.fromString("name"), getDatabaseName()), null); - String databaseId = ts.getObject(String.class); - if (databaseId != null) + TableSelector ts = new TableSelector(us.getTable("databases"), PageFlowUtil.set("objectid"), new SimpleFilter(FieldKey.fromString("name"), getDatabaseName()), new Sort("-rowid")); + if (ts.exists()) { + String databaseId = ts.getArrayList(String.class).stream().toList().get(0); getStatusLogger().info("jbrowse database exists using name: " + getDatabaseName()); boolean hadChanges = addTracks(databaseId, releaseId); @@ -84,34 +85,32 @@ protected Object doTransform(Object inputValue) return databaseId; } - else + + String databaseId = new GUID().toString(); + try { - try - { - databaseId = new GUID().toString(); - getStatusLogger().info("creating jbrowse database: " + databaseId + ", for output file: " + outputFileId); - - //create database - TableInfo databases = getJbrowseUserSchema().getTable("databases"); - CaseInsensitiveHashMap dbRow = new CaseInsensitiveHashMap<>(); - dbRow.put("objectid", databaseId); - dbRow.put("name", getDatabaseName()); - dbRow.put("description", null); - dbRow.put("libraryId", getLibraryId()); - dbRow.put("temporary", false); - dbRow.put("container", getContainerUser().getContainer().getId()); - dbRow.put("created", new Date()); - dbRow.put("createdby", getContainerUser().getUser().getUserId()); - dbRow.put("modified", new Date()); - dbRow.put("modifiedby", getContainerUser().getUser().getUserId()); - dbRow.put("jsonConfig", getSessionJson()); - - databases.getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(dbRow), new BatchValidationException(), null, new HashMap<>()); - } - catch (Exception e) - { - getStatusLogger().error("Error creating database: " + inputValue, e); - } + getStatusLogger().info("creating jbrowse database: " + databaseId + ", for output file: " + outputFileId); + + //create database + TableInfo databases = getJbrowseUserSchema().getTable("databases"); + CaseInsensitiveHashMap dbRow = new CaseInsensitiveHashMap<>(); + dbRow.put("objectid", databaseId); + dbRow.put("name", getDatabaseName()); + dbRow.put("description", null); + dbRow.put("libraryId", getLibraryId()); + dbRow.put("temporary", false); + dbRow.put("container", getContainerUser().getContainer().getId()); + dbRow.put("created", new Date()); + dbRow.put("createdby", getContainerUser().getUser().getUserId()); + dbRow.put("modified", new Date()); + dbRow.put("modifiedby", getContainerUser().getUser().getUserId()); + dbRow.put("jsonConfig", getSessionJson()); + + databases.getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(dbRow), new BatchValidationException(), null, new HashMap<>()); + } + catch (Exception e) + { + getStatusLogger().error("Error creating database: " + inputValue, e); } addTracks(databaseId, releaseId); From e683e40481b62185f51b94451a0baec2559a178d Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 11 May 2025 16:16:55 -0700 Subject: [PATCH 04/34] Conditionalize aligner index memory based on aligner type --- .../SequenceJobResourceAllocator.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index 34b7bbb7..1151b766 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -82,6 +82,24 @@ private boolean isSequenceSequenceOutputHandlerTask(PipelineJob job) private Long _totalFileSize = null; private static final Long UNABLE_TO_DETERMINE = -1L; + private int getAlignerIndexMem(PipelineJob job) + { + if (job instanceof HasJobParams) + { + Map params = ((HasJobParams)job).getJobParams(); + if (params.get("alignment") != null) + { + String aligner = params.get("alignment"); + if (Arrays.asList("BWA-Mem", "BWA-Mem2", "STAR").contains(aligner)) + { + return 72; + } + } + } + + return 36; + } + @Override public Integer getMaxRequestCpus(PipelineJob job) { @@ -165,11 +183,11 @@ public Integer getMaxRequestMemory(PipelineJob job) return 72; } - // NOTE: STAR in particular needs more memory if (isCacheAlignerIndexesTask(job)) { - job.getLogger().debug("setting memory to 36"); - return 36; + int mem = getAlignerIndexMem(job); + job.getLogger().debug("setting memory to: " + mem); + return mem; } if (isLuceneIndexJob(job)) From 31766871503a9c334035b00e6fbe4a5a515865e7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 13 May 2025 13:01:33 -0700 Subject: [PATCH 05/34] Improve mGapMaintenanceTask handling of symlinks --- .../org/labkey/mgap/mGapMaintenanceTask.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java index 373e4a30..450c0169 100644 --- a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java +++ b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -265,7 +266,23 @@ private void checkSymlink(Logger log, File f, @Nullable String dirName, List Date: Tue, 13 May 2025 15:06:30 -0700 Subject: [PATCH 06/34] Improve mGapMaintenanceTask handling of symlinks --- mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java index 450c0169..3f5231c3 100644 --- a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java +++ b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java @@ -219,7 +219,7 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U expectedFiles.add(new File(f.getPath() + ".tbi")); } - if ("sitesOnlyVcfId".equals(field)) + if ("vcfId".equals(field) || "sitesOnlyVcfId".equals(field)) { checkSymlink(log, f, releaseId, commandsToRun); checkSymlink(log, new File(f.getPath() + ".tbi"), releaseId, commandsToRun); From 7721673f51115bc383a59cc0122532f29149787f Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 17 May 2025 07:39:24 -0700 Subject: [PATCH 07/34] Bugfix to JBrowseSessionTransform --- .../mgap/columnTransforms/JBrowseSessionTransform.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java index 2ce9b755..04e11f67 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java @@ -217,7 +217,7 @@ protected boolean addTracks(final String databaseId, String releaseId) return hadChanges.get(); } - private void ensureLuceneData(String objectId) + private void ensureLuceneData(String objectId, boolean hasIndex) { //determine if there is already a JSONfile for this outputfile TableSelector ts1 = new TableSelector(getJsonFiles(), PageFlowUtil.set("container"), new SimpleFilter(FieldKey.fromString("objectid"), objectId), null); @@ -234,7 +234,7 @@ private void ensureLuceneData(String objectId) Map row = new CaseInsensitiveHashMap<>(); row.put("objectid", objectId); row.put("container", containerId); - row.put("trackJson", getTrackJson(true)); + row.put("trackJson", getTrackJson(hasIndex)); TableInfo jsonFiles = getJbrowseUserSchema().getTable("jsonfiles"); jsonFiles.getUpdateService().updateRows(getContainerUser().getUser(), getContainerUser().getContainer(), Arrays.asList(row), Arrays.asList(new CaseInsensitiveHashMap<>(Map.of("objectid", objectId))), new BatchValidationException(), null, null); @@ -335,7 +335,8 @@ private String getOrCreateJsonFile(Results rs, String fieldKey) throws SQLExcept String objectId = ts1.getArrayList(String.class).get(0); if (isDefaultTrack) { - ensureLuceneData(objectId); + boolean expectIndex = rs.getObject(FieldKey.fromString("releaseId/luceneIndex")) != null || rs.getObject(FieldKey.fromString("vcfIndexId")) != null; + ensureLuceneData(objectId, expectIndex); } return objectId; From e9feeddb95aa3d048e7f6b31ef1e8dbf646674c5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 18 May 2025 07:57:23 -0700 Subject: [PATCH 08/34] Update download docs --- mGAP/resources/web/mGAP/DownloadWindow.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mGAP/resources/web/mGAP/DownloadWindow.js b/mGAP/resources/web/mGAP/DownloadWindow.js index 69ff0a82..200f76f3 100644 --- a/mGAP/resources/web/mGAP/DownloadWindow.js +++ b/mGAP/resources/web/mGAP/DownloadWindow.js @@ -61,18 +61,18 @@ Ext4.define('mGAP.window.DownloadWindow', { var toAdd = [{ html: 'Due to the large file size, the preferred option is to download using wget or curl on the command line, such as the exmaples below. Nonetheless, you also are able to paste the URLs into your browser and download through this way as well, although it will be slower and possibly not able to resume if your connection is disrupted.

' + 'Use these to download the VCF and index:
' + - '
wget https://mgapdownload.ohsu.edu/' + releaseVcf + '
' +
-                            'wget https://mgapdownload.ohsu.edu/' + releaseVcf + '.tbi
' + + '
curl -C https://mgapdownload.ohsu.edu/' + releaseVcf + '
' +
+                            'curl -C https://mgapdownload.ohsu.edu/' + releaseVcf + '.tbi
' + (sitesOnlyVcf ? 'or a VCF without genotypes (considerably smaller):
' + - '
wget https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '
' +
-                            'wget https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '.tbi
' : '') + + '
curl -C https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '
' +
+                            'curl -C https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '.tbi
' : '') + 'and genome:
' + - '
wget https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '
' +
-                            'wget https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '.fai
' +
-                            'wget https://mgapdownload.ohsu.edu/genomes/' + urlFasta.replace(/fasta$/, 'dict') + '
' + '
curl -C https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '
' +
+                            'curl -C https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '.fai
' +
+                            'curl -C https://mgapdownload.ohsu.edu/genomes/' + urlFasta.replace(/fasta$/, 'dict') + '
' },{ - html: '
mGAP is an NIH funded project. If you use these data in a publication, we ask that you please include R24OD021324 in the acknowledgements.', + html: '
mGAP is an NIH funded project. If you use these data in a publication, we ask that you please include R24 OD021324 in the acknowledgements.', border: false, style: 'padding-bottom: 20px;' }]; From b5d84e7721f9af74d273e343d792d3f8dc63127d Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 18 May 2025 08:43:07 -0700 Subject: [PATCH 09/34] Improve curl syntax --- mGAP/resources/web/mGAP/DownloadWindow.js | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/mGAP/resources/web/mGAP/DownloadWindow.js b/mGAP/resources/web/mGAP/DownloadWindow.js index 200f76f3..f18dffef 100644 --- a/mGAP/resources/web/mGAP/DownloadWindow.js +++ b/mGAP/resources/web/mGAP/DownloadWindow.js @@ -58,19 +58,24 @@ Ext4.define('mGAP.window.DownloadWindow', { var urlFasta = results.rows[0]['vcfId/library_id/fasta_file/Name']; var sitesOnlyVcf = results.rows[0].objectId + '/' + results.rows[0]['sitesOnlyVcfId/dataid/Name']; + const getFileName = function(path){ + path = path.split('/'); + return(path[path.length - 1]); + } + var toAdd = [{ html: 'Due to the large file size, the preferred option is to download using wget or curl on the command line, such as the exmaples below. Nonetheless, you also are able to paste the URLs into your browser and download through this way as well, although it will be slower and possibly not able to resume if your connection is disrupted.

' + - 'Use these to download the VCF and index:
' + - '
curl -C https://mgapdownload.ohsu.edu/' + releaseVcf + '
' +
-                            'curl -C https://mgapdownload.ohsu.edu/' + releaseVcf + '.tbi
' + + 'Use these to download the VCF and index. Note, -C will allow file resume if the download is disrupted:
' + + '
curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/' + releaseVcf + '
' +
+                            'curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/' + releaseVcf + '.tbi
' + (sitesOnlyVcf ? 'or a VCF without genotypes (considerably smaller):
' + - '
curl -C https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '
' +
-                            'curl -C https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '.tbi
' : '') + + '
curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '
' +
+                            'curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '.tbi
' : '') + 'and genome:
' + - '
curl -C https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '
' +
-                            'curl -C https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '.fai
' +
-                            'curl -C https://mgapdownload.ohsu.edu/genomes/' + urlFasta.replace(/fasta$/, 'dict') + '
' + '
curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '
' +
+                            'curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '.fai
' +
+                            'curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/genomes/' + urlFasta.replace(/fasta$/, 'dict') + '
' },{ html: '
mGAP is an NIH funded project. If you use these data in a publication, we ask that you please include R24 OD021324 in the acknowledgements.', border: false, From e07a0845674a3935b3bc86ba45bd04df039a3a07 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 22 May 2025 15:15:48 -0700 Subject: [PATCH 10/34] Use mGapMaintenanceTask to create symlink alias for latest versions --- .../org/labkey/mgap/mGapMaintenanceTask.java | 106 ++++++++++++++++-- 1 file changed, 94 insertions(+), 12 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java index 3f5231c3..3ae7c4d0 100644 --- a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java +++ b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java @@ -1,5 +1,6 @@ package org.labkey.mgap; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; @@ -29,9 +30,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; public class mGapMaintenanceTask implements SystemMaintenance.MaintenanceTask @@ -90,6 +94,34 @@ private void checkMGapFiles(Logger log, User u) return; } + Map> versionMap = new HashMap<>(); + new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("species", "version", "objectid")).forEachResults(rs -> { + String species = rs.getString(FieldKey.fromString("species")); + if (!versionMap.containsKey(species)) + { + versionMap.put(species, new TreeMap<>()); + } + + String objectId = rs.getString(FieldKey.fromString("objectid")); + String versionString = rs.getString(FieldKey.fromString("version")); + if (NumberUtils.isCreatable(versionString)) + { + Double version = Double.parseDouble(versionString); + versionMap.get(species).put(version, objectId); + } + else + { + log.error("Non-numeric version: " + versionString); + } + }); + + Map latestReleaseBySpecies = new HashMap<>(); + for (String species : versionMap.keySet()) + { + Double maxVersion = versionMap.get(species).keySet().stream().sorted().max(Double::compareTo).get(); + latestReleaseBySpecies.put(species, versionMap.get(species).get(maxVersion)); + } + // Find expected folder names: List releaseIds = new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("objectid")).getArrayList(String.class); @@ -113,7 +145,7 @@ private void checkMGapFiles(Logger log, User u) } List commandsToRun = new ArrayList<>(); - releaseIds.forEach(f -> inspectReleaseFolder(f, baseDir, c, u, log, toDelete, commandsToRun)); + releaseIds.forEach(f -> inspectReleaseFolder(f, baseDir, c, u, log, toDelete, commandsToRun, latestReleaseBySpecies)); // Also verify genomes: Set genomesIds = new HashSet<>(new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("genomeId")).getArrayList(Integer.class)); @@ -151,7 +183,7 @@ private void checkMGapFiles(Logger log, User u) } } - private void inspectReleaseFolder(String releaseId, File baseDir, Container c, User u, final Logger log, final Set toDelete, List commandsToRun) + private void inspectReleaseFolder(String releaseId, File baseDir, Container c, User u, final Logger log, final Set toDelete, List commandsToRun, Map speciesToLatestTrack) { File releaseDir = new File(baseDir, releaseId); if (!releaseDir.exists()) @@ -160,6 +192,9 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U return; } + String species = new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("species"), new SimpleFilter(FieldKey.fromString("objectid"), releaseId), null).getObject(String.class); + boolean isLatestReleaseForSpecies = releaseId.equals(speciesToLatestTrack.get(species)); + final Set expectedFiles = new HashSet<>(); List tracksFromRelease = new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_TRACKS_PER_RELEASE), PageFlowUtil.set("vcfId"), new SimpleFilter(FieldKey.fromString("releaseId"), releaseId), null).getArrayList(Integer.class); tracksFromRelease.forEach(rowId -> { @@ -181,6 +216,11 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U checkSymlink(log, f, releaseId, commandsToRun); expectedFiles.add(new File(f.getPath() + ".tbi")); checkSymlink(log, new File(f.getPath() + ".tbi"), releaseId, commandsToRun); + + if (isLatestReleaseForSpecies) + { + checkSymlink(log, f, releaseId, commandsToRun); + } }); final Set fields = PageFlowUtil.set("vcfId", "variantTable", "liftedVcfId", "sitesOnlyVcfId", "novelSitesVcfId", "luceneIndex"); @@ -223,6 +263,14 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U { checkSymlink(log, f, releaseId, commandsToRun); checkSymlink(log, new File(f.getPath() + ".tbi"), releaseId, commandsToRun); + + if (isLatestReleaseForSpecies & "sitesOnlyVcfId".equals(field)) + { + File symlinkTarget = new File("/var/www/html/latest/" + ".mGAP." + species.replaceAll(" ", "_") + ".vcf.gz"); + checkSymlink(log, f, symlinkTarget, commandsToRun); + + checkSymlink(log, new File(f.getPath() + ".tbi"), new File(symlinkTarget.getPath() + ".tbi"), commandsToRun); + } } } }); @@ -263,27 +311,61 @@ private void checkSymlink(Logger log, File f, @Nullable String dirName, List commandsToRun) + { + if (!expectedSymlink.getParentFile().exists()) + { + commandsToRun.add("mkdir -p " + expectedSymlink.getParentFile()); + } + if (!expectedSymlink.exists()) { log.error("Missing symlink: " + expectedSymlink.getPath()); log.error("to path: " + f.getPath()); - File target = f; - if (Files.isSymbolicLink(target.toPath())) + File target = resolveSymlink(f); + commandsToRun.add("ln -s " + target.getPath() + " " + expectedSymlink.getPath()); + } + else + { + if (!Files.isSymbolicLink(expectedSymlink.toPath())) { - try - { - target = Files.readSymbolicLink(target.toPath()).toFile(); - log.error("which resolves to: " + target.getPath()); - } - catch (IOException e) + log.error("File is not a symlink: "+ expectedSymlink.getPath()); + } + else + { + File expectedTarget = resolveSymlink(f); + File actualTarget = resolveSymlink(expectedSymlink); + if (!expectedTarget.equals(actualTarget)) { - throw new RuntimeException(e); + log.error("Symlink does not point to the correct location: " + expectedSymlink.getPath()); + commandsToRun.add("rm " + expectedSymlink.getPath()); + + commandsToRun.add("ln -s " + expectedTarget.getPath() + " " + expectedSymlink.getPath()); } } + } + } - commandsToRun.add("ln -s " + target.getPath() + " " + expectedSymlink.getPath()); + private File resolveSymlink(File sourceFile) + { + File target = sourceFile; + if (Files.isSymbolicLink(target.toPath())) + { + try + { + target = Files.readSymbolicLink(target.toPath()).toFile(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } } + + return target; } private void checkForDuplicateAliases(Logger log, User u) From 82ef09f623a533f9662abd92dbf717c801355bb3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 23 May 2025 11:35:09 -0700 Subject: [PATCH 11/34] Bugfix name of symlink --- mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java index 3ae7c4d0..94aa3373 100644 --- a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java +++ b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java @@ -266,7 +266,7 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U if (isLatestReleaseForSpecies & "sitesOnlyVcfId".equals(field)) { - File symlinkTarget = new File("/var/www/html/latest/" + ".mGAP." + species.replaceAll(" ", "_") + ".vcf.gz"); + File symlinkTarget = new File("/var/www/html/latest/" + "mGAP." + species.replaceAll(" ", "_") + ".vcf.gz"); checkSymlink(log, f, symlinkTarget, commandsToRun); checkSymlink(log, new File(f.getPath() + ".tbi"), new File(symlinkTarget.getPath() + ".tbi"), commandsToRun); From 88f7ea5648d18d6f1fb48f604b0fade5b19f0815 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 28 May 2025 06:49:02 -0700 Subject: [PATCH 12/34] Add link to mGAP dashboard --- mGAP/resources/views/mgapDataDashboard.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mGAP/resources/views/mgapDataDashboard.html b/mGAP/resources/views/mgapDataDashboard.html index 71a53780..87cf98a9 100644 --- a/mGAP/resources/views/mgapDataDashboard.html +++ b/mGAP/resources/views/mgapDataDashboard.html @@ -168,6 +168,14 @@ schemaName: 'mgap', queryName: 'duplicategvcfs' }) + },{ + name: 'CRAMs Not Using Archival Mode', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, { + schemaName: 'sequenceanalysis', + queryName: 'outputfiles', + 'query.dataid/DataFileUrl~contains': '.cram', + 'query.description~doesnotcontain': 'Archival' + }) }] }] }] From 847dcefaaf1920141ef0355d549b50a4d10e90ca Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 3 Jun 2025 06:24:30 -0700 Subject: [PATCH 13/34] Update ETL order and npm dependencies --- mcc/package-lock.json | 6 +++--- mcc/resources/etls/mcc.xml | 34 +++++++++++++++++----------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mcc/package-lock.json b/mcc/package-lock.json index 7ad10c48..542c41cf 100644 --- a/mcc/package-lock.json +++ b/mcc/package-lock.json @@ -8157,9 +8157,9 @@ } }, "node_modules/tar-fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", - "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", + "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", "dev": true, "dependencies": { "chownr": "^1.1.1", diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index f9379952..9e3d71ad 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -95,23 +95,6 @@ - - Copy to target - - - Id - date - category - observation - objectid - - - - - - - - Copy to target @@ -153,6 +136,23 @@ + + Copy to target + + + Id + date + category + observation + objectid + + + + + + + + From 4c62ac7c7be1f10f28d7e418acb48e1498cc624a Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 6 Jun 2025 10:42:00 -0700 Subject: [PATCH 14/34] Add draft notification --- .../org/labkey/primeseq/PrimeseqModule.java | 4 + .../notification/DiskUsageNotification.java | 149 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index af4095cf..7bf969e3 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -23,6 +23,7 @@ import org.labkey.api.data.Container; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.ldk.LDKService; +import org.labkey.api.ldk.notification.NotificationService; import org.labkey.api.module.ModuleContext; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; @@ -32,6 +33,7 @@ import org.labkey.api.view.WebPartFactory; import org.labkey.primeseq.analysis.CombineMethylationRatesHandler; import org.labkey.primeseq.analysis.MethylationRateComparisonHandler; +import org.labkey.primeseq.notification.DiskUsageNotification; import org.labkey.primeseq.pipeline.BismarkWrapper; import org.labkey.primeseq.pipeline.BlastPipelineJobResourceAllocator; import org.labkey.primeseq.pipeline.ClusterMaintenanceTask; @@ -85,6 +87,8 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) LDKService.get().registerQueryButton(new UpdateResourcesButton(), "pipeline", "job"); LDKService.get().registerQueryButton(new PerformMhcCleanupButton(), "sequenceanalysis", "sequence_analyses"); LDKService.get().registerQueryButton(new DeleteJobCheckpointButton(), "pipeline", "job"); + + NotificationService.get().registerNotification(new DiskUsageNotification()); } @Override diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java new file mode 100644 index 00000000..62189200 --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -0,0 +1,149 @@ +package org.labkey.primeseq.notification; + +import org.apache.commons.lang3.SystemUtils; +import org.apache.commons.lang3.time.DurationFormatUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.ldk.notification.Notification; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.api.settings.LookAndFeelProperties; + +import java.text.DateFormat; +import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * User: bbimber + * Date: 8/4/12 + */ +public class DiskUsageNotification implements Notification +{ + protected final static Logger _log = LogManager.getLogger(DiskUsageNotification.class); + private static final String lastSave = "lastSave"; + private NumberFormat _pctFormat = null; + + private static final String PROP_CATEGORY = "primeseq.DiskUsageNotification"; + + @Override + public String getName() + { + return "Disk/Cluster Usage Notification"; + } + + @Override + public String getCategory() + { + return "Admin"; + } + + @Override + public boolean isAvailable(Container c) + { + return true; + } + + @Override + public String getDescription() + { + return "This runs once a week to summarize file system and cluster usage"; + } + + @Override + public String getEmailSubject(Container c) + { + return "Disk/Cluster Usage: " + getDateTimeFormat(c).format(new Date()); + } + + public DateFormat getDateTimeFormat(Container c) + { + return new SimpleDateFormat(LookAndFeelProperties.getInstance(c).getDefaultDateTimeFormat()); + } + + @Override + public String getCronString() + { + return "0 8 * * 1 ?"; + } + + @Override + public String getScheduleDescription() + { + return "Every Monday at 8AM"; + } + + private Map getSavedValues(Container c) + { + return PropertyManager.getProperties(c, PROP_CATEGORY); + } + + @Override + public String getMessageBodyHTML(Container c, User u) + { + Date start = new Date(); + + _pctFormat = NumberFormat.getPercentInstance(); + _pctFormat.setMaximumFractionDigits(1); + + Map saved = getSavedValues(c); + Map newValues = new HashMap<>(); + + StringBuilder msg = new StringBuilder(); + StringBuilder alerts = new StringBuilder(); + + getDiskUsageStats(c, u, msg); + + if (alerts.length() > 0) + { + alerts.insert(0, "The following alerts were generated:

"); + alerts.append("

"); + msg.insert(0, alerts); + } + + msg.insert(0, "This email contains a series of alerts designed for site admins. It was run on: " + getDateTimeFormat(c).format(new Date()) + ". Runtime: " + DurationFormatUtils.formatDurationWords((new Date()).getTime() - start.getTime(), true, true) + "

"); + + return msg.toString(); + } + + private void getDiskUsageStats(Container c, User u, final StringBuilder msg) + { + if (!SystemUtils.IS_OS_LINUX) + { + return; + } + + try + { + SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); + + String results = wrapper.executeWithOutput(Arrays.asList("df", "-h", "/home/groups/BimberLab/", "/home/groups/OnprcColonyData/", "/home/groups/prime-seq/", "/home/exacloud/gscratch/prime-seq/")); + + msg.append("Disk Usage Stats:

"); + msg.append(""); + Arrays.stream(results.split("\n")).forEach(x -> { + msg.append(""); + Arrays.stream(x.split("[ ]+")).forEach(cell -> { + msg.append(""); + }); + + msg.append(""); + }); + msg.append("
"); + msg.append(cell); + msg.append("
"); + } + catch (PipelineJobException e) + { + _log.error("Error running df", e); + } + + msg.append("

\n"); + } +} \ No newline at end of file From 9559abce88c5ee81daaeca8a455a325e3d7c107a Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 6 Jun 2025 13:11:35 -0700 Subject: [PATCH 15/34] Improve formatting --- .../notification/DiskUsageNotification.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 62189200..66a64426 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -100,7 +100,7 @@ public String getMessageBodyHTML(Container c, User u) getDiskUsageStats(c, u, msg); - if (alerts.length() > 0) + if (!alerts.isEmpty()) { alerts.insert(0, "The following alerts were generated:

"); alerts.append("

"); @@ -126,15 +126,20 @@ private void getDiskUsageStats(Container c, User u, final StringBuilder msg) String results = wrapper.executeWithOutput(Arrays.asList("df", "-h", "/home/groups/BimberLab/", "/home/groups/OnprcColonyData/", "/home/groups/prime-seq/", "/home/exacloud/gscratch/prime-seq/")); msg.append("Disk Usage Stats:

"); - msg.append(""); + msg.append("
"); Arrays.stream(results.split("\n")).forEach(x -> { - msg.append(""); - Arrays.stream(x.split("[ ]+")).forEach(cell -> { - msg.append(""); - }); + String[] els = x.split("[ ]+"); + if ("Filesystem".equalsIgnoreCase(els[0])) + { + return; + } + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); msg.append(""); }); msg.append("
FilesystemSizeUsedAvailablePercent
"); - msg.append(cell); - msg.append("
").append(els[0]).append("").append(els[1]).append("").append(els[2]).append("").append(els[3]).append("").append(els[4]).append("
"); From 36f5e7aa021e75b5ae2a66ed6f2a04527b36b00a Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 7 Jun 2025 08:12:31 -0700 Subject: [PATCH 16/34] Expand DiskUsageNotification --- .../notification/DiskUsageNotification.java | 61 +++++++++++++++---- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 66a64426..baefb0f3 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -19,6 +19,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; /** * User: bbimber @@ -92,24 +93,61 @@ public String getMessageBodyHTML(Container c, User u) _pctFormat = NumberFormat.getPercentInstance(); _pctFormat.setMaximumFractionDigits(1); - Map saved = getSavedValues(c); - Map newValues = new HashMap<>(); - StringBuilder msg = new StringBuilder(); - StringBuilder alerts = new StringBuilder(); - getDiskUsageStats(c, u, msg); + getClusterUsage(c, u, msg); + + msg.insert(0, "This email summarizes disk and cluster usage. It was run on: " + getDateTimeFormat(c).format(new Date()) + ". Runtime: " + DurationFormatUtils.formatDurationWords((new Date()).getTime() - start.getTime(), true, true) + "

"); + + return msg.toString(); + } - if (!alerts.isEmpty()) + private void getClusterUsage(Container c, User u, final StringBuilder msg) + { + if (!SystemUtils.IS_OS_LINUX) { - alerts.insert(0, "The following alerts were generated:

"); - alerts.append("

"); - msg.insert(0, alerts); + return; } - msg.insert(0, "This email contains a series of alerts designed for site admins. It was run on: " + getDateTimeFormat(c).format(new Date()) + ". Runtime: " + DurationFormatUtils.formatDurationWords((new Date()).getTime() - start.getTime(), true, true) + "

"); + try + { + SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); + String results = wrapper.executeWithOutput(Arrays.asList("ssh", "labkey_submit@arc", "sshare", "-U", "-u", "labkey_submit")); + + msg.append("Cluster Usage:

"); + msg.append(""); + + AtomicBoolean foundHeader = new AtomicBoolean(false); + Arrays.stream(results.split("\n")).forEach(x -> { + if (x.startsWith("------------")) + { + foundHeader.set(true); + return; + } + else if (!foundHeader.get()) + { + return; + } + + String[] els = x.split("[ ]+"); + + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + }); + msg.append("
AccountRawSharesNormSharesRawUsageEffectiveUsageFairShare
").append(els[0]).append("").append(els[3]).append("").append(els[4]).append("").append(els[5]).append("").append(els[6]).append("
"); + } + catch (PipelineJobException e) + { + _log.error("Error running df", e); + } + + msg.append("

\n"); - return msg.toString(); } private void getDiskUsageStats(Container c, User u, final StringBuilder msg) @@ -122,7 +160,6 @@ private void getDiskUsageStats(Container c, User u, final StringBuilder msg) try { SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); - String results = wrapper.executeWithOutput(Arrays.asList("df", "-h", "/home/groups/BimberLab/", "/home/groups/OnprcColonyData/", "/home/groups/prime-seq/", "/home/exacloud/gscratch/prime-seq/")); msg.append("Disk Usage Stats:

"); From 04a313feed8d0717ea2d2403a8a58cd15809d9fc Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 7 Jun 2025 10:04:54 -0700 Subject: [PATCH 17/34] Suppress ssh banners --- .../primeseq/notification/DiskUsageNotification.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index baefb0f3..0cc8868b 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -1,5 +1,6 @@ package org.labkey.primeseq.notification; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.logging.log4j.LogManager; @@ -112,7 +113,7 @@ private void getClusterUsage(Container c, User u, final StringBuilder msg) try { SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); - String results = wrapper.executeWithOutput(Arrays.asList("ssh", "labkey_submit@arc", "sshare", "-U", "-u", "labkey_submit")); + String results = wrapper.executeWithOutput(Arrays.asList("ssh", "-q", "labkey_submit@arc", "sshare", "-U", "-u", "labkey_submit")); msg.append("Cluster Usage:

"); msg.append(""); @@ -131,6 +132,12 @@ else if (!foundHeader.get()) String[] els = x.split("[ ]+"); + if (els.length != 7) + { + _log.error("Unexpected line: " + StringUtils.join(els, "<>")); + return; + } + msg.append(""); msg.append(""); msg.append(""); @@ -143,7 +150,7 @@ else if (!foundHeader.get()) } catch (PipelineJobException e) { - _log.error("Error running df", e); + _log.error("Error fetching slurm summary", e); } msg.append("

\n"); From f2ef1fff52bf9815885978bf674c65b4631f5dbf Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 7 Jun 2025 13:21:19 -0700 Subject: [PATCH 18/34] Remove field from table --- .../labkey/primeseq/notification/DiskUsageNotification.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 0cc8868b..7245af41 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -116,7 +116,7 @@ private void getClusterUsage(Container c, User u, final StringBuilder msg) String results = wrapper.executeWithOutput(Arrays.asList("ssh", "-q", "labkey_submit@arc", "sshare", "-U", "-u", "labkey_submit")); msg.append("Cluster Usage:

"); - msg.append("

AccountRawSharesNormSharesRawUsageEffectiveUsageFairShare
").append(els[0]).append("").append(els[3]).append("
"); + msg.append("
AccountRawSharesNormSharesRawUsageEffectiveUsageFairShare
"); AtomicBoolean foundHeader = new AtomicBoolean(false); Arrays.stream(results.split("\n")).forEach(x -> { @@ -141,7 +141,8 @@ else if (!foundHeader.get()) msg.append(""); msg.append(""); msg.append(""); - msg.append(""); + long shares = StringUtils.isEmpty(els[4]) ? 0 : Long.parseLong(els[4]); + msg.append(""); msg.append(""); msg.append(""); msg.append(""); From 3890848783b5155a3345051bec8296e10f77759c Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 9 Jun 2025 16:15:46 -0700 Subject: [PATCH 19/34] New ETL method to sync data based on a source of subject IDs --- PMR/resources/etls/pmr-datasets.xml | 3 +- .../labkey/pmr/etl/SubjectScopedSelect.java | 442 ++++++++++++++++++ 2 files changed, 443 insertions(+), 2 deletions(-) create mode 100644 PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml index 1186b27a..a0fc89c5 100644 --- a/PMR/resources/etls/pmr-datasets.xml +++ b/PMR/resources/etls/pmr-datasets.xml @@ -189,8 +189,7 @@ - - + diff --git a/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java b/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java new file mode 100644 index 00000000..35b4ee37 --- /dev/null +++ b/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java @@ -0,0 +1,442 @@ +package org.labkey.pmr.etl; + +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationError; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.writer.ContainerUser; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class SubjectScopedSelect implements TaskRefTask +{ + protected final Map _settings = new CaseInsensitiveHashMap<>(); + protected ContainerUser _containerUser; + + private enum Settings + { + subjectRemoteSource(false), + subjectSourceContainerPath(true), + subjectSourceSchema(true), + subjectSourceQuery(true), + subjectSourceColumn(true), + + dataRemoteSource(false), + dataSourceContainerPath(true), + dataSourceSchema(true), + dataSourceQuery(true), + dataSourceColumns(true), + dataSourceColumnMapping(false), + dataSourceAdditionalFilters(false), + + targetSchema(true), + targetQuery(true), + targetSubjectColumn(true), + targetAdditionalFilters(false); + + private final boolean _isRequired; + + Settings(boolean isRequired) + { + _isRequired = isRequired; + } + + public boolean isRequired() + { + return _isRequired; + } + } + + final int BATCH_SIZE = 500; + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + List subjects = getSubjects(job.getLogger()); + List> batches = Lists.partition(subjects, BATCH_SIZE); + job.getLogger().info("Total batches: " + batches.size()); + batches.forEach(x -> processBatch(x, job.getLogger())); + + return new RecordedActionSet(); + } + + private void processBatch(List subjects, Logger log) + { + log.info("processing batch with " + subjects.size() + " subjects"); + TableInfo destinationTable = getDataDestinationTable(); + + QueryUpdateService qus = destinationTable.getUpdateService(); + qus.setBulkLoad(true); + + try + { + // Find / Delete existing values: + Set keyFields = destinationTable.getColumns().stream().filter(ColumnInfo::isKeyField).collect(Collectors.toSet()); + final SimpleFilter subjectFilter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name())), subjects, CompareType.IN); + if (_settings.get(Settings.targetAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.targetAdditionalFilters.name())); + additionalFilters.forEach(subjectFilter::addCondition); + } + + Collection> existingRows = new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection(); + if (!existingRows.isEmpty()) + { + log.info("deleting " + existingRows.size() + " rows"); + qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), new ArrayList<>(existingRows), null, null); + } + else + { + log.info("No rows to delete for this subject batch"); + } + + // Query data and import + List> toImport = getRowsToImport(log); + if (!toImport.isEmpty()) + { + log.info("inserting " + toImport.size() + " rows"); + qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImport, new BatchValidationException(), null, null); + } + else + { + log.info("No rows to import for this subject batch"); + } + } + catch (SQLException | InvalidKeyException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e) + { + throw new IllegalStateException("Error Importing Rows", e); + } + } + + private List parseAdditionalFilters(String rawVal) + { + rawVal = StringUtils.trimToNull(rawVal); + if (rawVal == null) + { + return Collections.emptyList(); + } + + SimpleFilter filter = new SimpleFilter(); + String[] filters = rawVal.split(","); + for (String queryParam : filters) + { + filter.addUrlFilters(new ActionURL().setRawQuery(queryParam), null); + } + + return filter.getClauses().stream().map(fc -> { + if (fc instanceof CompareType.CompareClause cc) + { + return cc; + } + + throw new IllegalStateException("Expected all filters to be instance CompareType.CompareClause, found: " + fc.getClass()); + }).toList(); + } + + private Map parseSourceToDestColumnMap(String rawVal) + { + if (rawVal == null) + { + return Collections.emptyMap(); + } + + Map colMap = new HashMap<>(); + + String[] tokens = rawVal.split(","); + for (String token : tokens) + { + if (!token.contains("=")) + { + throw new IllegalStateException("Invalid columnMapping: " + token); + } + + String[] els = token.split("="); + if (els.length != 2) + { + throw new IllegalStateException("Invalid columnMapping: " + token); + } + + colMap.put(els[0], els[1]); + } + + return colMap; + } + + private List> getRowsToImport(Logger log) + { + if (_settings.get(Settings.dataSourceColumns.name()) == null) + { + throw new IllegalStateException("Missing value for dataSourceColumns"); + } + List sourceColumns = Arrays.asList(_settings.get(Settings.dataSourceColumns.name()).split(",")); + + Map sourceToDestColumMap = parseSourceToDestColumnMap(_settings.get(Settings.dataSourceColumnMapping.name())); + + if (_settings.get(Settings.dataRemoteSource.name()) != null) + { + DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.dataRemoteSource.name()), log); + SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.dataSourceSchema.name()), _settings.get(Settings.dataSourceQuery.name())); + sr.setColumns(sourceColumns); + if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name())); + for (CompareType.CompareClause f : additionalFilters) + { + Object value; + if (f.getParamVals() == null) + { + value = null; + } + else if (f.getParamVals().length == 1) + { + value = f.getParamVals()[0]; + } + else + { + value = StringUtils.join(f.getParamVals(), ";"); + } + + sr.addFilter(new Filter(f.getFieldKey().toString(), value, Filter.Operator.valueOf(f.getCompareType().getFilterValueText()))); + } + } + + try + { + SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer); + + return doNameMapping(srr.getRows(), sourceToDestColumMap); + } + catch (CommandException | IOException e) + { + throw new IllegalStateException(e); + } + } + else + { + Container source = ContainerManager.getForPath(_settings.get(Settings.dataSourceContainerPath.name())); + if (source == null) + { + throw new IllegalStateException("Unknown container: " + _settings.get(Settings.dataSourceContainerPath.name())); + } + + if (!source.hasPermission(_containerUser.getUser(), ReadPermission.class)) + { + throw new IllegalStateException("User does not have read permission: " + _settings.get(Settings.dataSourceContainerPath.name())); + } + + UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), source, _settings.get(Settings.dataSourceSchema.name())); + if (us == null) + { + throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.dataSourceSchema.name())); + } + + TableInfo sourceTable = us.getTable(_settings.get(Settings.dataSourceQuery.name())); + if (sourceTable == null) + { + throw new IllegalStateException("Unknown table: " + _settings.get(Settings.dataSourceQuery.name())); + } + + if (!sourceTable.hasPermission(_containerUser.getUser(), ReadPermission.class)) + { + throw new IllegalStateException("User does not have read permission on the source table " + _settings.get(Settings.dataSourceContainerPath.name())); + } + + for (String colName : sourceColumns) + { + if (sourceTable.getColumn(colName) == null) + { + throw new IllegalStateException("Table is missing column: " + colName); + } + } + + + final SimpleFilter filter = new SimpleFilter(); + if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name())); + additionalFilters.forEach(filter::addCondition); + } + + TableSelector ts = new TableSelector(sourceTable, PageFlowUtil.set(_settings.get(Settings.subjectSourceColumn.name())), filter, null); + + return doNameMapping(new ArrayList<>(ts.getMapCollection()), sourceToDestColumMap); + } + } + + private List> doNameMapping(List> rows, Map colMap) + { + return rows.stream().map(row -> { + if (colMap.isEmpty()) + { + return row; + } + + colMap.forEach((sourceCol, destCol) -> { + if (row.containsKey(sourceCol)) + { + row.put(destCol, row.get(sourceCol)); + } + + row.remove(sourceCol); + }); + + return row; + }).toList(); + } + + private TableInfo getDataDestinationTable() + { + UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), _settings.get(Settings.targetSchema.name())); + if (us == null) + { + throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.targetSchema.name())); + } + + TableInfo sourceTable = us.getTable(_settings.get(Settings.targetQuery.name())); + if (sourceTable == null) + { + throw new IllegalStateException("Unknown table: " + _settings.get(Settings.targetQuery.name())); + } + + return sourceTable; + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } + + @Override + public List preFlightCheck(Container c) + { + List errors = new ArrayList<>(); + for (String setting : getRequiredSettings()) + { + if (_settings.get(setting) == null) + { + errors.add(new SimpleValidationError("Missing required setting: " + setting)); + } + } + + return errors; + } + + @Override + public List getRequiredSettings() + { + return Arrays.stream(Settings.values()).filter(Settings::isRequired).map(Settings::name).toList(); + } + + @Override + public void setSettings(Map settings) + { + _settings.putAll(settings); + } + + private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Logger log) throws IllegalStateException + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, _containerUser.getContainer(), log); + if (rc == null) + { + throw new IllegalStateException("Unable to find remote connection: " + name); + } + + return rc; + } + + private List getSubjects(Logger log) + { + if (_settings.get(Settings.subjectRemoteSource.name()) != null) + { + DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.subjectRemoteSource.name()), log); + SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.subjectSourceSchema.name()), _settings.get(Settings.subjectSourceQuery.name())); + sr.setColumns(Arrays.asList(_settings.get(Settings.subjectSourceColumn.name()))); + + try + { + SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer); + + return srr.getRows().stream().map(x -> x.get(_settings.get(Settings.subjectSourceColumn.name()))).map(Object::toString).toList(); + } + catch (CommandException | IOException e) + { + throw new IllegalStateException(e); + } + } + else + { + Container source = ContainerManager.getForPath(_settings.get(Settings.subjectSourceContainerPath.name())); + if (source == null) + { + throw new IllegalStateException("Unknown container: " + _settings.get(Settings.subjectSourceContainerPath.name())); + } + + if (!source.hasPermission(_containerUser.getUser(), ReadPermission.class)) + { + throw new IllegalStateException("User does not have read permission: " + _settings.get(Settings.subjectSourceContainerPath.name())); + } + + UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), source, _settings.get(Settings.subjectSourceSchema.name())); + if (us == null) + { + throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.subjectSourceSchema.name())); + } + + TableInfo sourceTable = us.getTable(_settings.get(Settings.subjectSourceQuery.name())); + if (sourceTable == null) + { + throw new IllegalStateException("Unknown table: " + _settings.get(Settings.subjectSourceQuery.name())); + } + + if (sourceTable.getColumn(_settings.get(Settings.subjectSourceColumn.name())) == null) + { + throw new IllegalStateException("Table is missing column: " + _settings.get(Settings.subjectSourceColumn.name())); + } + + return new TableSelector(sourceTable, PageFlowUtil.set(_settings.get(Settings.subjectSourceColumn.name()))).getArrayList(String.class); + } + } + + +} From 510810c3d1f2ced72958b994fffb91514141cd42 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 9 Jun 2025 16:50:34 -0700 Subject: [PATCH 20/34] Add new ETL type to sync based on subject ID --- PMR/resources/etls/siv-studies.xml | 116 ++++++++++++++++++ .../labkey/pmr/etl/SubjectScopedSelect.java | 59 ++++++++- 2 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 PMR/resources/etls/siv-studies.xml diff --git a/PMR/resources/etls/siv-studies.xml b/PMR/resources/etls/siv-studies.xml new file mode 100644 index 00000000..449c4e3e --- /dev/null +++ b/PMR/resources/etls/siv-studies.xml @@ -0,0 +1,116 @@ + + + SIV_PRIMe + SIV Studies / PRIMe Data + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java b/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java index 35b4ee37..1c44bca3 100644 --- a/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java +++ b/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java @@ -56,18 +56,19 @@ public class SubjectScopedSelect implements TaskRefTask private enum Settings { subjectRemoteSource(false), - subjectSourceContainerPath(true), + subjectSourceContainerPath(false), subjectSourceSchema(true), subjectSourceQuery(true), subjectSourceColumn(true), dataRemoteSource(false), - dataSourceContainerPath(true), + dataSourceContainerPath(false), dataSourceSchema(true), dataSourceQuery(true), dataSourceColumns(true), dataSourceColumnMapping(false), dataSourceAdditionalFilters(false), + dataSourceColumnDefaults(false), targetSchema(true), targetQuery(true), @@ -175,6 +176,7 @@ private List parseAdditionalFilters(String rawVal) private Map parseSourceToDestColumnMap(String rawVal) { + rawVal = StringUtils.trimToNull(rawVal); if (rawVal == null) { return Collections.emptyMap(); @@ -202,6 +204,36 @@ private Map parseSourceToDestColumnMap(String rawVal) return colMap; } + private Map parseColumnDefaultMap(String rawVal) + { + rawVal = StringUtils.trimToNull(rawVal); + if (rawVal == null) + { + return Collections.emptyMap(); + } + + Map colMap = new HashMap<>(); + + String[] tokens = rawVal.split(";"); + for (String token : tokens) + { + if (!token.contains("=")) + { + throw new IllegalStateException("Invalid column defaultValue: " + token); + } + + String[] els = token.split("="); + if (els.length != 2) + { + throw new IllegalStateException("Invalid column defaultValue: " + token); + } + + colMap.put(els[0], els[1]); + } + + return colMap; + } + private List> getRowsToImport(Logger log) { if (_settings.get(Settings.dataSourceColumns.name()) == null) @@ -211,6 +243,7 @@ private List> getRowsToImport(Logger log) List sourceColumns = Arrays.asList(_settings.get(Settings.dataSourceColumns.name()).split(",")); Map sourceToDestColumMap = parseSourceToDestColumnMap(_settings.get(Settings.dataSourceColumnMapping.name())); + Map columnToDefaultMap = parseColumnDefaultMap(_settings.get(Settings.dataSourceColumnDefaults.name())); if (_settings.get(Settings.dataRemoteSource.name()) != null) { @@ -244,7 +277,7 @@ else if (f.getParamVals().length == 1) { SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer); - return doNameMapping(srr.getRows(), sourceToDestColumMap); + return doNameMapping(srr.getRows(), sourceToDestColumMap, columnToDefaultMap); } catch (CommandException | IOException e) { @@ -253,6 +286,10 @@ else if (f.getParamVals().length == 1) } else { + if (_settings.get(Settings.dataSourceContainerPath.name()) == null) + { + throw new IllegalStateException("Must provide dataSourceContainerPath for local sources"); + } Container source = ContainerManager.getForPath(_settings.get(Settings.dataSourceContainerPath.name())); if (source == null) { @@ -299,11 +336,11 @@ else if (f.getParamVals().length == 1) TableSelector ts = new TableSelector(sourceTable, PageFlowUtil.set(_settings.get(Settings.subjectSourceColumn.name())), filter, null); - return doNameMapping(new ArrayList<>(ts.getMapCollection()), sourceToDestColumMap); + return doNameMapping(new ArrayList<>(ts.getMapCollection()), sourceToDestColumMap, columnToDefaultMap); } } - private List> doNameMapping(List> rows, Map colMap) + private List> doNameMapping(List> rows, Map colMap, Map columnToDefaultMap) { return rows.stream().map(row -> { if (colMap.isEmpty()) @@ -320,6 +357,16 @@ private List> doNameMapping(List> rows, row.remove(sourceCol); }); + return row; + }).map(row -> { + for (String colName : columnToDefaultMap.keySet()) + { + if (!row.containsKey(colName)) + { + row.put(colName, columnToDefaultMap.get(colName)); + } + } + return row; }).toList(); } @@ -409,7 +456,7 @@ private List getSubjects(Logger log) Container source = ContainerManager.getForPath(_settings.get(Settings.subjectSourceContainerPath.name())); if (source == null) { - throw new IllegalStateException("Unknown container: " + _settings.get(Settings.subjectSourceContainerPath.name())); + source = _containerUser.getContainer(); } if (!source.hasPermission(_containerUser.getUser(), ReadPermission.class)) From 0aa74989023ecea83ce8bace6bebeff13ed56c2e Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 9 Jun 2025 17:19:42 -0700 Subject: [PATCH 21/34] Make IDs unique --- PMR/resources/etls/siv-studies.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PMR/resources/etls/siv-studies.xml b/PMR/resources/etls/siv-studies.xml index 449c4e3e..470028d9 100644 --- a/PMR/resources/etls/siv-studies.xml +++ b/PMR/resources/etls/siv-studies.xml @@ -66,7 +66,7 @@ - + From 8beb68a37dbf83f32f514091b2faa86c943cb0b1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 10 Jun 2025 10:35:58 -0700 Subject: [PATCH 22/34] Refactor code into discrete SivStudies module --- SivStudies/build.gradle | 20 + SivStudies/module.properties | 7 + SivStudies/resources/data/amount_units.tsv | 11 + .../data/calculated_status_codes.tsv | 6 + SivStudies/resources/data/conc_units.tsv | 12 + SivStudies/resources/data/dosage_units.tsv | 11 + SivStudies/resources/data/gender_codes.tsv | 4 + .../resources/data/geographic_origins.tsv | 4 + SivStudies/resources/data/lookup_sets.tsv | 9 + SivStudies/resources/data/reports.tsv | 12 + SivStudies/resources/data/routes.tsv | 24 ++ SivStudies/resources/data/species.tsv | 10 + SivStudies/resources/data/volume_units.tsv | 13 + .../resources/etls/siv-studies.xml | 10 +- .../folderTypes/SIV Studies.folderType.xml | 78 ++++ .../queries/study/assignment.query.xml | 31 ++ .../queries/study/demographics.query.xml | 65 +++ .../resources/queries/study/flags.query.xml | 22 + .../queries/study/genetics.query.xml | 30 ++ .../queries/study/immunizations.query.xml | 33 ++ .../resources/queries/study/labwork.query.xml | 35 ++ .../queries/study/procedures.query.xml | 22 + .../resources/queries/study/samples.query.xml | 31 ++ .../queries/study/studyData.query.xml | 25 ++ .../queries/study/treatments.query.xml | 72 ++++ .../queries/study/viralloads.query.xml | 42 ++ .../resources/queries/study/weight.query.xml | 20 + .../resources/referenceStudy/folder.xml | 4 + .../study/datasets/Studies.dataset | 19 + .../study/datasets/datasets_manifest.xml | 41 ++ .../study/datasets/datasets_metadata.xml | 392 ++++++++++++++++++ .../resources/referenceStudy/study/study.xml | 9 + .../referenceStudy/study/studyPolicy.xml | 10 + .../postgresql/sivstudies-25.000-25.001.sql | 19 + SivStudies/resources/schemas/sivstudies.xml | 2 + SivStudies/resources/views/studiesAdmin.html | 33 ++ .../resources/views/studiesAdmin.view.xml | 8 + .../resources/views/studiesAdmin.webpart.xml | 6 + .../sivstudies/SivStudiesController.java | 104 +++++ .../labkey/sivstudies/SivStudiesManager.java | 32 ++ .../labkey/sivstudies/SivStudiesModule.java | 70 ++++ .../labkey/sivstudies/SivStudiesSchema.java | 47 +++ .../query/SivStudiesCustomizer.java | 99 +++++ .../test/tests/sivstudies/SivStudiesTest.java | 86 ++++ 44 files changed, 1635 insertions(+), 5 deletions(-) create mode 100644 SivStudies/build.gradle create mode 100644 SivStudies/module.properties create mode 100644 SivStudies/resources/data/amount_units.tsv create mode 100644 SivStudies/resources/data/calculated_status_codes.tsv create mode 100644 SivStudies/resources/data/conc_units.tsv create mode 100644 SivStudies/resources/data/dosage_units.tsv create mode 100644 SivStudies/resources/data/gender_codes.tsv create mode 100644 SivStudies/resources/data/geographic_origins.tsv create mode 100644 SivStudies/resources/data/lookup_sets.tsv create mode 100644 SivStudies/resources/data/reports.tsv create mode 100644 SivStudies/resources/data/routes.tsv create mode 100644 SivStudies/resources/data/species.tsv create mode 100644 SivStudies/resources/data/volume_units.tsv rename {PMR => SivStudies}/resources/etls/siv-studies.xml (94%) create mode 100644 SivStudies/resources/folderTypes/SIV Studies.folderType.xml create mode 100644 SivStudies/resources/queries/study/assignment.query.xml create mode 100644 SivStudies/resources/queries/study/demographics.query.xml create mode 100644 SivStudies/resources/queries/study/flags.query.xml create mode 100644 SivStudies/resources/queries/study/genetics.query.xml create mode 100644 SivStudies/resources/queries/study/immunizations.query.xml create mode 100644 SivStudies/resources/queries/study/labwork.query.xml create mode 100644 SivStudies/resources/queries/study/procedures.query.xml create mode 100644 SivStudies/resources/queries/study/samples.query.xml create mode 100644 SivStudies/resources/queries/study/studyData.query.xml create mode 100644 SivStudies/resources/queries/study/treatments.query.xml create mode 100644 SivStudies/resources/queries/study/viralloads.query.xml create mode 100644 SivStudies/resources/queries/study/weight.query.xml create mode 100644 SivStudies/resources/referenceStudy/folder.xml create mode 100644 SivStudies/resources/referenceStudy/study/datasets/Studies.dataset create mode 100644 SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml create mode 100644 SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml create mode 100644 SivStudies/resources/referenceStudy/study/study.xml create mode 100644 SivStudies/resources/referenceStudy/study/studyPolicy.xml create mode 100644 SivStudies/resources/schemas/dbscripts/postgresql/sivstudies-25.000-25.001.sql create mode 100644 SivStudies/resources/schemas/sivstudies.xml create mode 100644 SivStudies/resources/views/studiesAdmin.html create mode 100644 SivStudies/resources/views/studiesAdmin.view.xml create mode 100644 SivStudies/resources/views/studiesAdmin.webpart.xml create mode 100644 SivStudies/src/org/labkey/sivstudies/SivStudiesController.java create mode 100644 SivStudies/src/org/labkey/sivstudies/SivStudiesManager.java create mode 100644 SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java create mode 100644 SivStudies/src/org/labkey/sivstudies/SivStudiesSchema.java create mode 100644 SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java create mode 100644 SivStudies/test/src/org/labkey/test/tests/sivstudies/SivStudiesTest.java diff --git a/SivStudies/build.gradle b/SivStudies/build.gradle new file mode 100644 index 00000000..68d68d56 --- /dev/null +++ b/SivStudies/build.gradle @@ -0,0 +1,20 @@ +import org.labkey.gradle.util.BuildUtils + +plugins { + id 'org.labkey.build.module' +} + +dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:premiumModules:dataintegration", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module") + + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:premiumModules:dataintegration", depProjectConfig: "apiJarFile") + + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") +} diff --git a/SivStudies/module.properties b/SivStudies/module.properties new file mode 100644 index 00000000..142ebb1a --- /dev/null +++ b/SivStudies/module.properties @@ -0,0 +1,7 @@ +ModuleClass: org.labkey.sivstudies.SivStudiesModule +Label: SIV Studies +Description: A module designed to manage data from SIV and related studies. +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: false +SupportedDatabases: mssql, pgsql diff --git a/SivStudies/resources/data/amount_units.tsv b/SivStudies/resources/data/amount_units.tsv new file mode 100644 index 00000000..b4b01026 --- /dev/null +++ b/SivStudies/resources/data/amount_units.tsv @@ -0,0 +1,11 @@ +value +g +IU +U +mEq +mg +no units +ug +units +L/min +% \ No newline at end of file diff --git a/SivStudies/resources/data/calculated_status_codes.tsv b/SivStudies/resources/data/calculated_status_codes.tsv new file mode 100644 index 00000000..d2bf1dec --- /dev/null +++ b/SivStudies/resources/data/calculated_status_codes.tsv @@ -0,0 +1,6 @@ +value +Alive +Dead +No Record +Shipped +Unknown \ No newline at end of file diff --git a/SivStudies/resources/data/conc_units.tsv b/SivStudies/resources/data/conc_units.tsv new file mode 100644 index 00000000..d7d9b4d1 --- /dev/null +++ b/SivStudies/resources/data/conc_units.tsv @@ -0,0 +1,12 @@ +value denominator numerator +g/ml mL g +g/tsp tsp g +IU/ml mL IU +mEq/ml mL mEq +mg/capsule capsule(s) mg +mg/ml mL mg +mg/piece piece(s) mg +mg/tablet tablet(s) mg +mg/tsp tsp mg +ug/ml mL ug +units/ml mL units \ No newline at end of file diff --git a/SivStudies/resources/data/dosage_units.tsv b/SivStudies/resources/data/dosage_units.tsv new file mode 100644 index 00000000..eb8de149 --- /dev/null +++ b/SivStudies/resources/data/dosage_units.tsv @@ -0,0 +1,11 @@ +value numerator denominator +g/kg g kg +IU/kg IU kg +mEq/kg mEq kg +mg/animal mg +mg/kg mg kg +ml/kg ml kg +no units +ounces/kg ounces kg +ug/kg ug kg +units/kg units kg \ No newline at end of file diff --git a/SivStudies/resources/data/gender_codes.tsv b/SivStudies/resources/data/gender_codes.tsv new file mode 100644 index 00000000..07b68195 --- /dev/null +++ b/SivStudies/resources/data/gender_codes.tsv @@ -0,0 +1,4 @@ +v meaning origgender +f Female f +m Male m +u Unknown \ No newline at end of file diff --git a/SivStudies/resources/data/geographic_origins.tsv b/SivStudies/resources/data/geographic_origins.tsv new file mode 100644 index 00000000..0870abf8 --- /dev/null +++ b/SivStudies/resources/data/geographic_origins.tsv @@ -0,0 +1,4 @@ +value +Indian +Chinese +Hybrid \ No newline at end of file diff --git a/SivStudies/resources/data/lookup_sets.tsv b/SivStudies/resources/data/lookup_sets.tsv new file mode 100644 index 00000000..8e2b9482 --- /dev/null +++ b/SivStudies/resources/data/lookup_sets.tsv @@ -0,0 +1,9 @@ +setname label keyfield titleColumn +amount_units Amount Units unit +calculated_status_codes Calculated Status Codes code +conc_units Concentraiton Units unit +dosage_units Dosage Units unit +gender_codes Gender Codes +geographic_origins Geographic Origins origin +routes Routes route +volume_units Volume Units unit \ No newline at end of file diff --git a/SivStudies/resources/data/reports.tsv b/SivStudies/resources/data/reports.tsv new file mode 100644 index 00000000..4dd48af5 --- /dev/null +++ b/SivStudies/resources/data/reports.tsv @@ -0,0 +1,12 @@ +reportname category reporttype reporttitle visible containerpath schemaname queryname viewname report datefieldname todayonly queryhaslocation sort_order QCStateLabelFieldName description +activeAssignments Assignments and Groups query Active Assignments true study Assignment Active Assignments date false false qcstate/publicdata This report shows the active assignments for each animal +assignmentHistory Assignments and Groups query Assignment History true study Assignment date false false qcstate/publicdata This report shows all assignments records for the animals +activeGroups Assignments and Groups query Active Groups true study animal_group_members Active Members date false false qcstate/publicdata This report shows the active assignments for each animal +groupHistory Assignments and Groups query Group History true study animal_group_members date false false qcstate/publicdata This report shows all assignments records for the animals +microbiology Lab Results query Microbiology true study Microbiology Results date false false qcstate/publicdata +biochemistry Lab Results js Biochemistry true study bloodChemistry date false false Contains results of chemistry panels. Can be displayed either by panel, or showing reference ranges +clinPathRuns Lab Results query Lab Runs true study Clinpath Runs date false false qcstate/publicdata Contains all clinpath requests +iStat Lab Results js iStat true study iStat date false false qcstate/publicdata Contains iStat results +hematology Lab Results js Hematology true study hematology date false false Contains hematology data showing cell subsets +parasitology Lab Results query Parasitology true study Parasitology Results date false false qcstate/publicdata Contains results of parasitology testing +urinalysis Lab Results js Urinalysis true study urinalysisResults date false false Contains urinalysis results diff --git a/SivStudies/resources/data/routes.tsv b/SivStudies/resources/data/routes.tsv new file mode 100644 index 00000000..b0e1f0d8 --- /dev/null +++ b/SivStudies/resources/data/routes.tsv @@ -0,0 +1,24 @@ +value title +IM +intracardiac +intracarotid +intracorneal Intracorneal +intracranial +IP intraperitoneal +ID +INH +IT +IV +CRI +IVAG +oral +PO +rectal +Spillage +SQ +OU +OD +OS +topical +topical (eye) +topical (skin) \ No newline at end of file diff --git a/SivStudies/resources/data/species.tsv b/SivStudies/resources/data/species.tsv new file mode 100644 index 00000000..0e063c4c --- /dev/null +++ b/SivStudies/resources/data/species.tsv @@ -0,0 +1,10 @@ +common scientific_name id_prefix mhc_prefix blood_per_kg max_draw_pct blood_draw_interval cites_code dateDisabled +Baboon 60 0.2 30 +Cotton-top Tamarin Saguinus oedipus so Saoe 60 0.2 30 +Cynomolgus Macaca fascicularis cy Mafa 60 0.2 30 +Marmoset Callithrix jacchus cj Caja 60 0.15 30 +Pigtail Macaca Nemestrina Mane 60 0.2 30 +Rhesus Macaca mulatta r|rh Mamu 60 0.2 30 +Sooty Mangabey Cercocebus atys Ceat 60 0.2 30 +Stump Tailed Macaca Arctoides Maar 60 0.2 30 +Vervet Chlorocebus sabaeus ag Chsa 60 0.2 30 diff --git a/SivStudies/resources/data/volume_units.tsv b/SivStudies/resources/data/volume_units.tsv new file mode 100644 index 00000000..fc2c7682 --- /dev/null +++ b/SivStudies/resources/data/volume_units.tsv @@ -0,0 +1,13 @@ +value +capsule(s) +cup +drop(s) +cube +mL +mL/hr +no units +ounce(s) +pan +piece(s) +tablet(s) +tsp \ No newline at end of file diff --git a/PMR/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml similarity index 94% rename from PMR/resources/etls/siv-studies.xml rename to SivStudies/resources/etls/siv-studies.xml index 470028d9..0d1f5772 100644 --- a/PMR/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -4,7 +4,7 @@ SIV Studies / PRIMe Data - + @@ -23,7 +23,7 @@ - + @@ -44,7 +44,7 @@ - + @@ -67,7 +67,7 @@ - + @@ -90,7 +90,7 @@ - + diff --git a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml new file mode 100644 index 00000000..749c8d8f --- /dev/null +++ b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml @@ -0,0 +1,78 @@ + + Studies Overview + The default folder layout for Studies + + + + + + + + + + + + + + + + + datasets + Datasets + + + datasets + + + + + + + Datasets + body + + + + + + + + + + + + + + + + + + + + + + admin + Admin + + + + + + + + + Studies Admin + body + + + + + + + + + studies + + studies + true + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml new file mode 100644 index 00000000..05d40e8c --- /dev/null +++ b/SivStudies/resources/queries/study/assignment.query.xml @@ -0,0 +1,31 @@ + + + +
AccountNormSharesRawUsageEffectiveUsageFairShare
").append(els[0]).append("").append(els[3]).append("").append(els[4]).append("").append(String.format("%,d", shares)).append("").append(els[5]).append("").append(els[6]).append("
+ + + + Date Added + Date + + + End Date + Date + + + Study + + + Sub-Group + + + Cohort ID + + + Category + + +
+ + + diff --git a/SivStudies/resources/queries/study/demographics.query.xml b/SivStudies/resources/queries/study/demographics.query.xml new file mode 100644 index 00000000..f06d781f --- /dev/null +++ b/SivStudies/resources/queries/study/demographics.query.xml @@ -0,0 +1,65 @@ + + + + + + + + + + + true + false + + + Date + Birth + /query/executeQuery.view? + schemaName=study& + query.queryName=Birth& + query.Id~eq=${Id} + + + + Date + Death + /query/executeQuery.view? + schemaName=study& + query.queryName=Deaths& + query.Id~eq=${Id} + + + + true + false + true + + + Sex + + + Geographic Origin + + + Birth + + + Death + + + Species + + + Mother + + + Father + + + Status + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/flags.query.xml b/SivStudies/resources/queries/study/flags.query.xml new file mode 100644 index 00000000..6e2b23ed --- /dev/null +++ b/SivStudies/resources/queries/study/flags.query.xml @@ -0,0 +1,22 @@ + + + + + + + + Date Added + Date + + + End Date + Date + + + Flag + + +
+ + + diff --git a/SivStudies/resources/queries/study/genetics.query.xml b/SivStudies/resources/queries/study/genetics.query.xml new file mode 100644 index 00000000..54b08dae --- /dev/null +++ b/SivStudies/resources/queries/study/genetics.query.xml @@ -0,0 +1,30 @@ + + + + + + + + Date Added + Date + + + Category + + + Assay Type + + + Marker/Allele + + + Result + + + Score + + +
+ + + diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml new file mode 100644 index 00000000..833d0fde --- /dev/null +++ b/SivStudies/resources/queries/study/immunizations.query.xml @@ -0,0 +1,33 @@ + + + + + + + + Date Added + Date + + + Category + + + Treatment + + + Route + + + Quantity + + + Quantity Units + + + Reason + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/labwork.query.xml b/SivStudies/resources/queries/study/labwork.query.xml new file mode 100644 index 00000000..99e85a14 --- /dev/null +++ b/SivStudies/resources/queries/study/labwork.query.xml @@ -0,0 +1,35 @@ + + + + + + + + Date + Date + + + Category + + + Test + + + Result + + + Units + + + Qualitative Result + true + false + + + Method + + +
+ + + diff --git a/SivStudies/resources/queries/study/procedures.query.xml b/SivStudies/resources/queries/study/procedures.query.xml new file mode 100644 index 00000000..b866f8b5 --- /dev/null +++ b/SivStudies/resources/queries/study/procedures.query.xml @@ -0,0 +1,22 @@ + + + + + + + + + + true + + + Category + + + Procedure + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml new file mode 100644 index 00000000..87ac5494 --- /dev/null +++ b/SivStudies/resources/queries/study/samples.query.xml @@ -0,0 +1,31 @@ + + + + + + + + Date Added + Date + + + Sample ID + + + Sample Type + + + Preservation + + + Quantity + + + Quantity Units + + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml new file mode 100644 index 00000000..bf1d995a --- /dev/null +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + Date + + + End Date + + + Key + true + false + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/treatments.query.xml b/SivStudies/resources/queries/study/treatments.query.xml new file mode 100644 index 00000000..1a5e82c9 --- /dev/null +++ b/SivStudies/resources/queries/study/treatments.query.xml @@ -0,0 +1,72 @@ + + + + + + + + Date + Date + + + End Date + Date + + + Category + + + Treatment + + + Route + + + Frequency + + + Amount + + + Amount Units + + + Volume + true + false + + + Volume Units + true + false + + + Concentration + true + false + + + Conc. Units + true + false + + + Dosage + true + false + + + Dosage Units + true + false + + + Reason + true + false + + +
+ + + diff --git a/SivStudies/resources/queries/study/viralloads.query.xml b/SivStudies/resources/queries/study/viralloads.query.xml new file mode 100644 index 00000000..ef00f0e2 --- /dev/null +++ b/SivStudies/resources/queries/study/viralloads.query.xml @@ -0,0 +1,42 @@ + + + + + + + + Date + Date + + + Sample Type + + + Assay Type + + + target + + + LOD + + + Result + + + Units + + + Qualitative Result + true + false + + + true + false + + +
+ + + diff --git a/SivStudies/resources/queries/study/weight.query.xml b/SivStudies/resources/queries/study/weight.query.xml new file mode 100644 index 00000000..24e484dd --- /dev/null +++ b/SivStudies/resources/queries/study/weight.query.xml @@ -0,0 +1,20 @@ + + + + + + + + + + true + + + Weight (kg) + 0.#### + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/referenceStudy/folder.xml b/SivStudies/resources/referenceStudy/folder.xml new file mode 100644 index 00000000..e3acbb15 --- /dev/null +++ b/SivStudies/resources/referenceStudy/folder.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/SivStudies/resources/referenceStudy/study/datasets/Studies.dataset b/SivStudies/resources/referenceStudy/study/datasets/Studies.dataset new file mode 100644 index 00000000..8e597028 --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/datasets/Studies.dataset @@ -0,0 +1,19 @@ +# default group can be used to avoid repeating definitions for each dataset +# +# action=[REPLACE,APPEND,DELETE] (default:REPLACE) +# deleteAfterImport=[TRUE|FALSE] (default:FALSE) + +default.action=REPLACE +default.deleteAfterImport=FALSE + +# map a source tsv column (right side) to a property name or full propertyURI (left) +# predefined properties: ParticipantId, SiteId, VisitId, Created +default.property.ParticipantId=ptid +default.property.Created=dfcreate + +# use to map from filename->datasetid +# NOTE: if there are NO explicit import definitions, we will try to import all files matching pattern +# NOTE: if there are ANY explicit mapping, we will only import listed datasets + +default.filePattern=dataset(\\d*).tsv +default.importAllMatches=TRUE diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml new file mode 100644 index 00000000..346bd030 --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml new file mode 100644 index 00000000..ecd9b819 --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -0,0 +1,392 @@ + + + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + timestamp + + + varchar + + + Flags/Misc Information +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + timestamp + + + varchar + + + varchar + + + double + + + varchar + + + double + + + varchar + + + double + + + varchar + + + varchar + + + integer + + + double + + + varchar + + + varchar + + + Medications/Treatments +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + varchar + + + varchar + + + varchar + + + double + + + varchar + + + varchar + + + Immunizations +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + double + + + Weight +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + varchar + + + varchar + + + varchar + + + double + + + double + + + varchar + true + + org.labkey.studies.query.ResultsOOODisplayColumn + + + + varchar + + + varchar + + + Viral Loads +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + varchar + + + double + + + varchar + + + varchar + + + varchar + + + varchar + + + Lab Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + false + + + + varchar + + + varchar + + + timestamp + + + timestamp + + + varchar + + + varchar + + + varchar + + + varchar + + + Demographics +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + timestamp + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + Project Assignment +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + varchar + + + varchar + + + varchar + + + double + + + varchar + + + Samples +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + varchar + + + varchar + + + varchar + + + varchar + + + double + + + Genetic Data +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + varchar + + + varchar + + + Procedures +
+ diff --git a/SivStudies/resources/referenceStudy/study/study.xml b/SivStudies/resources/referenceStudy/study/study.xml new file mode 100644 index 00000000..0acc6d7d --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/study.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/SivStudies/resources/referenceStudy/study/studyPolicy.xml b/SivStudies/resources/referenceStudy/study/studyPolicy.xml new file mode 100644 index 00000000..3755d25b --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/studyPolicy.xml @@ -0,0 +1,10 @@ + + + BASIC_WRITE + + + + + + + \ No newline at end of file diff --git a/SivStudies/resources/schemas/dbscripts/postgresql/sivstudies-25.000-25.001.sql b/SivStudies/resources/schemas/dbscripts/postgresql/sivstudies-25.000-25.001.sql new file mode 100644 index 00000000..cbf0c4f4 --- /dev/null +++ b/SivStudies/resources/schemas/dbscripts/postgresql/sivstudies-25.000-25.001.sql @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for SivStudies module here +-- All SQL VIEW definitions should be created in sivstudies-create.sql and dropped in sivstudies-drop.sql +CREATE SCHEMA sivstudies; diff --git a/SivStudies/resources/schemas/sivstudies.xml b/SivStudies/resources/schemas/sivstudies.xml new file mode 100644 index 00000000..f2708fff --- /dev/null +++ b/SivStudies/resources/schemas/sivstudies.xml @@ -0,0 +1,2 @@ + \ No newline at end of file diff --git a/SivStudies/resources/views/studiesAdmin.html b/SivStudies/resources/views/studiesAdmin.html new file mode 100644 index 00000000..b17398a9 --- /dev/null +++ b/SivStudies/resources/views/studiesAdmin.html @@ -0,0 +1,33 @@ + \ No newline at end of file diff --git a/SivStudies/resources/views/studiesAdmin.view.xml b/SivStudies/resources/views/studiesAdmin.view.xml new file mode 100644 index 00000000..198818fc --- /dev/null +++ b/SivStudies/resources/views/studiesAdmin.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/SivStudies/resources/views/studiesAdmin.webpart.xml b/SivStudies/resources/views/studiesAdmin.webpart.xml new file mode 100644 index 00000000..21f7c9a6 --- /dev/null +++ b/SivStudies/resources/views/studiesAdmin.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java new file mode 100644 index 00000000..ea678411 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.sivstudies; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.resource.Resource; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.studies.StudiesService; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.HtmlView; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.servlet.ModelAndView; + +public class SivStudiesController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SivStudiesController.class); + public static final String NAME = "sivstudies"; + + private static final Logger _log = LogHelper.getLogger(SivStudiesController.class, "SivStudiesController messages"); + + public SivStudiesController() + { + setActionResolver(_actionResolver); + } + + @RequiresPermission(AdminPermission.class) + public static class ImportStudyAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + setTitle("Import Study"); + + return new HtmlView(HtmlString.unsafe("This will import the default study in this folder, and truncate/load ancillary data. Do you want to continue?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + StudiesService.get().importFolderDefinition(getContainer(), getUser(), ModuleLoader.getInstance().getModule(SivStudiesModule.NAME), new Path("referenceStudy")); + + Module m = ModuleLoader.getInstance().getModule(SivStudiesModule.NAME); + StudiesService.get().loadTsv(m.getModuleResource("data/lookup_sets.tsv"), SivStudiesSchema.NAME, getUser(), getContainer()); + + Resource r = m.getModuleResource("data"); + r.list().forEach(tsv -> { + if ("lookup_sets.tsv".equals(tsv.getName())) + { + return; + } + + String schemaName = switch (tsv.getName()) + { + case "reports.tsv" -> "laboratory"; + case "species.tsv" -> "laboratory"; + default -> SivStudiesSchema.NAME; + }; + + StudiesService.get().loadTsv(tsv, schemaName, getUser(), getContainer()); + }); + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + + } + + @NotNull + @Override + public URLHelper getSuccessURL(Object o) + { + return PageFlowUtil.urlProvider(PipelineUrls.class).urlBegin(getContainer()); + } + } +} diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesManager.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesManager.java new file mode 100644 index 00000000..9682dadb --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesManager.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.sivstudies; + +public class SivStudiesManager +{ + private static final SivStudiesManager _instance = new SivStudiesManager(); + + private SivStudiesManager() + { + // prevent external construction with a private default constructor + } + + public static SivStudiesManager get() + { + return _instance; + } +} \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java new file mode 100644 index 00000000..020e9889 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.sivstudies; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.ldk.ExtendedSimpleModule; +import org.labkey.api.module.ModuleContext; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +public class SivStudiesModule extends ExtendedSimpleModule +{ + public static final String NAME = "SivStudies"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 25.001; + } + + @Override + protected void init() + { + addController(SivStudiesController.NAME, SivStudiesController.class); + } + + @Override + public void doStartupAfterSpringConfig(ModuleContext moduleContext) + { + + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return Collections.singleton(SivStudiesSchema.NAME); + } +} \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesSchema.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesSchema.java new file mode 100644 index 00000000..608fbdf1 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesSchema.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.sivstudies; + +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.dialect.SqlDialect; + +public class SivStudiesSchema +{ + private static final SivStudiesSchema _instance = new SivStudiesSchema(); + public static final String NAME = "sivstudies"; + + public static SivStudiesSchema getInstance() + { + return _instance; + } + + private SivStudiesSchema() + { + + } + + public DbSchema getSchema() + { + return DbSchema.get(NAME, DbSchemaType.Module); + } + + public SqlDialect getSqlDialect() + { + return getSchema().getSqlDialect(); + } +} diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java new file mode 100644 index 00000000..5fba56d0 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -0,0 +1,99 @@ +package org.labkey.sivstudies.query; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.TableInfo; +import org.labkey.api.ldk.table.AbstractTableCustomizer; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.User; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.DatasetTable; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; + +public class SivStudiesCustomizer extends AbstractTableCustomizer +{ + @Override + public void customize(TableInfo tableInfo) + { + if (tableInfo instanceof DatasetTable ds) + { + performDatasetCustomization(ds); + } + } + + public void performDatasetCustomization(DatasetTable ds) + { + _log.debug("Customizing: {}", ds.getName()); + + if (!ds.getDataset().isDemographicData()) + { + if (ds instanceof AbstractTableInfo ati) + { + addAgeAtTimeCol(ati, "date"); + } + else + { + _log.error("Expected DatasetTable to be instanceof AbstractTableInfo"); + } + } + } + + private @Nullable String getStudyDemographicsSchemaTableName(Container c, User u) + { + Study s = StudyService.get().getStudy(c); + if (s == null) + { + return null; + } + + Dataset ds = s.getDatasetByLabel("demographics"); + if (ds == null) + { + return null; + } + + return ds.getDomain().getStorageTableName(); + } + + private void addAgeAtTimeCol(AbstractTableInfo ti, String dateColName) + { + final String name = "ageAtTime"; + if (ti.getColumn(name, false) != null) + { + return; + } + + final String demographicsTableName = getStudyDemographicsSchemaTableName(ti.getUserSchema().getContainer(), ti.getUserSchema().getUser()); + final FieldKey birthFieldKey = FieldKey.fromString("Id/demographics/birth"); + + //new SQLFragment("(SELECT .birth FROM studydatasets." + getStudyDemographicsSchemaTableName()) + + // TODO + } + + private void appendMhcColumns(AbstractTableInfo ti, String dateColName) + { + + } + + private void appendPvlColumns(AbstractTableInfo ti, ColumnInfo subjectCol, ColumnInfo dateCol) + { + Container target = ti.getUserSchema().getContainer().isWorkbookOrTab() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer(); + +// TableInfo data = schema.createDataTable(null); +// String tableName = data.getDomain().getStorageTableName(); +// +// String name = "viralLoad"; +// if (ti.getColumn(name) == null) +// { +// SQLFragment sql = new SQLFragment("(SELECT avg(viralLoad) as expr FROM assayresult." + tableName + " t WHERE t.subjectId = " + ExprColumn.STR_TABLE_ALIAS + "." + subjectCol.getName() + " AND CAST(t.date AS DATE) = CAST(" + ExprColumn.STR_TABLE_ALIAS + "." + dateCol.getName() + " AS DATE))"); +// ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.DOUBLE, subjectCol, dateCol); +// newCol.setDescription("Displays the viral load from this timepoint, if present"); +// newCol.setLabel("Viral Load (copies/mL)"); +// ti.addColumn(newCol); +// } + } +} diff --git a/SivStudies/test/src/org/labkey/test/tests/sivstudies/SivStudiesTest.java b/SivStudies/test/src/org/labkey/test/tests/sivstudies/SivStudiesTest.java new file mode 100644 index 00000000..59575f2c --- /dev/null +++ b/SivStudies/test/src/org/labkey/test/tests/sivstudies/SivStudiesTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.tests.sivstudies; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.TestTimeoutException; +import org.labkey.test.categories.InDevelopment; + +import java.util.Collections; +import java.util.List; + +/** + * ~ Created from 'moduleTemplate' ~ + * This test validates basic functionality of the SivStudies module + */ +@Category({InDevelopment.class}) +public class SivStudiesTest extends BaseWebDriverTest +{ + @Override + protected void doCleanup(boolean afterTest) throws TestTimeoutException + { + _containerHelper.deleteProject(getProjectName(), afterTest); + } + + @BeforeClass + public static void setupProject() + { + SivStudiesTest init = getCurrentTest(); + + init.doSetup(); + } + + private void doSetup() + { + _containerHelper.createProject(getProjectName(), null); + } + + @Before + public void preTest() + { + goToProjectHome(); + } + + @Test + public void testSivStudiesModule() + { + _containerHelper.enableModule("SivStudies"); + + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } + + @Override + protected String getProjectName() + { + return "SivStudiesTest Project"; + } + + @Override + public List getAssociatedModules() + { + return Collections.singletonList("SivStudies"); + } +} \ No newline at end of file From 6382ab16d16e62f1e2bba805573505bcfd0add3d Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 10 Jun 2025 13:08:22 -0700 Subject: [PATCH 23/34] Correct URL for track --- mGAP/resources/views/overview.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/resources/views/overview.html b/mGAP/resources/views/overview.html index 65ada0ff..9da48179 100644 --- a/mGAP/resources/views/overview.html +++ b/mGAP/resources/views/overview.html @@ -46,7 +46,7 @@

Structural Variant Catalog: We have released a draft dataset with structural variants generated from PacBio sequencing of 44 Rhesus macaques. These data complement the short variant catalog by detecting categories of variants not readily accomplished with short read Illumina data.

NHP Models of Human Disease: Phenotypes

From df263c7a11c1d7489e164cd4d0b6dd3828584903 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 11 Jun 2025 05:15:05 -0700 Subject: [PATCH 24/34] Refactor several TCR tables --- .../cdna_libraries/Assay Info.qview.xml | 2 - .../postgresql/tcrdb-15.52-15.53.sql | 23 + .../dbscripts/sqlserver/tcrdb-15.52-15.53.sql | 23 + tcrdb/resources/schemas/tcrdb.xml | 591 +----------------- tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 2 +- tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java | 2 + tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java | 1 + 7 files changed, 65 insertions(+), 579 deletions(-) create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.52-15.53.sql create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql diff --git a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml index f64b4c07..2ac1c14c 100644 --- a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml +++ b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml @@ -18,7 +18,6 @@ - @@ -27,6 +26,5 @@ - \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.52-15.53.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.52-15.53.sql new file mode 100644 index 00000000..cd444bfe --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.52-15.53.sql @@ -0,0 +1,23 @@ +drop table tcrdb.stims; +drop table tcrdb.sorts; +drop table tcrdb.cdnas; +drop table tcrdb.peptides; +drop table tcrdb.plate_processing; +drop table tcrdb.citeseq_panels; +drop table tcrdb.citeseq_antibodies; + +ALTER TABLE tcrdb.clone_responses DROP COLUMN experiment; +ALTER TABLE tcrdb.clone_responses DROP COLUMN cloneName; +ALTER TABLE tcrdb.clone_responses DROP COLUMN date; +ALTER TABLE tcrdb.clone_responses DROP COLUMN cellBackground; +ALTER TABLE tcrdb.clone_responses DROP COLUMN numEffectors; +ALTER TABLE tcrdb.clone_responses DROP COLUMN pctTransduction; +ALTER TABLE tcrdb.clone_responses DROP COLUMN costim; +ALTER TABLE tcrdb.clone_responses DROP COLUMN antigen; +ALTER TABLE tcrdb.clone_responses DROP COLUMN stim; + +ALTER TABLE tcrdb.clone_responses ADD COLUMN cdna_id int; +ALTER TABLE tcrdb.clone_responses ADD COLUMN nostimid int; +ALTER TABLE tcrdb.clone_responses ADD COLUMN chain varchar(100); +ALTER TABLE tcrdb.clone_responses ADD COLUMN clonotype varchar(1000); + diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql new file mode 100644 index 00000000..cd444bfe --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql @@ -0,0 +1,23 @@ +drop table tcrdb.stims; +drop table tcrdb.sorts; +drop table tcrdb.cdnas; +drop table tcrdb.peptides; +drop table tcrdb.plate_processing; +drop table tcrdb.citeseq_panels; +drop table tcrdb.citeseq_antibodies; + +ALTER TABLE tcrdb.clone_responses DROP COLUMN experiment; +ALTER TABLE tcrdb.clone_responses DROP COLUMN cloneName; +ALTER TABLE tcrdb.clone_responses DROP COLUMN date; +ALTER TABLE tcrdb.clone_responses DROP COLUMN cellBackground; +ALTER TABLE tcrdb.clone_responses DROP COLUMN numEffectors; +ALTER TABLE tcrdb.clone_responses DROP COLUMN pctTransduction; +ALTER TABLE tcrdb.clone_responses DROP COLUMN costim; +ALTER TABLE tcrdb.clone_responses DROP COLUMN antigen; +ALTER TABLE tcrdb.clone_responses DROP COLUMN stim; + +ALTER TABLE tcrdb.clone_responses ADD COLUMN cdna_id int; +ALTER TABLE tcrdb.clone_responses ADD COLUMN nostimid int; +ALTER TABLE tcrdb.clone_responses ADD COLUMN chain varchar(100); +ALTER TABLE tcrdb.clone_responses ADD COLUMN clonotype varchar(1000); + diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index a6dd79ed..027dc8c3 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -97,476 +97,36 @@ - +
- TCR Stims/Samples - rowid - - - Stim Id - - - Tube # - - - Animal/Cell - http://cpas.labkey.com/Study#ParticipantId - false - - - Tissue - true - - - Effectors - false - - - # Effectors - - - APCs - true - - - # APCs - true - - - Sample Date - http://cpas.labkey.com/laboratory#sampleDate - false - Date - - - Peptide/Stim - false - - tcrdb - peptides - stim - - - - - - - - Co-stim - - - Background Freq - - - Activated Freq - - - - - - - - - lsidtype - true - true - false - - ObjectUri - Object - exp - - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - - true - - - TCR Sorts + TCR Clone Responsesrowid - Sort Id - false - - - Stim Id - - tcrdb - stims - rowid - - false - - - false - - - - - - # Cells - false - - - true - - - false - - laboratory - 96well_plate - well - - - - - Hashtag Oligo - - tcrdb - hashtag_oligos - tag_name - - - - - - - - - - - lsidtype - true - true - false - - ObjectUri - Object - exp - - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 + Row Id true - -
- - - TCR cDNA Libraries - rowid - - - Library Id - false - - - Sort Id - - tcrdb - sorts - rowid - - false - - - - - - - - - false - - - false - - laboratory - 96well_plate - well - - - - - Readset (GEX) + + cDNA ID - sequenceanalysis - sequence_readsets + singlecell + cdna_libraries rowid - - Readset (TCR) + + No Stim / Control cDNA ID - sequenceanalysis - sequence_readsets + singlecell + cdna_libraries rowid - - Readset (Cell Hashing) - - sequenceanalysis - sequence_readsets - rowid - - - - Readset (CITE-Seq) - - sequenceanalysis - sequence_readsets - rowid - - - - cDNA Concentration (ng/uL) - - - TCR Enriched Concentration (ng/uL) - - - Cite-Seq Panel - - tcrdb - citeseq_panel_names - name - - - - - - - - - - - lsidtype - true - true - false - - ObjectUri - Object - exp - - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - - - laboratory/buttons.js - - - - Laboratory.buttonHandlers.importDataHandlerForView(dataRegionName, 'tcrdb', 'stimDashboard') - - - Laboratory.buttonHandlers.importDataHandlerForView(dataRegionName, 'tcrdb', 'poolImport') - - - - - - -
- - - Peptides/Stims - - - Peptide/Stim - - - Category - - - Type - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - Processing Needed/Plate - - - Row Id - - - Plate Id - - - Processing Needed - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - - TCR Clone Responses - rowid - - - Row Id - true - - - Experiment - - - Clone Name - true - - - Cell Background - - - Date - http://cpas.labkey.com/laboratory#sampleDate - Date - - - # Effectors - - - Pct Transduction - - - Peptide/Stim - false - - - Co-stim + + Chain - - Antigen + + Clonotype Activated Freq @@ -685,125 +245,6 @@ false false false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - CITE-Seq Panels - - - Row Id - true - - - Panel Name - - - Marker/Antigen - - tcrdb - citeseq_antibodies - antibodyName - - - - - Marker/Antigen Label In Panel - If the desired label is different than the default label for this marker, enter a value - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - CITE-Seq Antibodies - - - Row Id - true - - - Antibody Name - - - Marker/Antigen Name - - - Marker/Antigen Label - - - Clone Name - - - Vendor - - - Product Id - - - Barcode Name - - sequenceanalysis - barcodes - tag_name - - - - - Adapter Sequence - textarea - - - true - - - true - - - false - false - false - true - 29 true @@ -813,8 +254,6 @@ false false false - true - 29 true diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 6d5f552f..c6ec96f3 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -46,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.52; + return 15.53; } @Override diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java index ebde2946..8f460121 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java @@ -69,6 +69,8 @@ public List getDataNavItems(Container c, User u) } items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, "TCR Clones", LaboratoryService.NavItemCategory.data, NAME, cache)); + items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONE_RESPONSES, "TCR Clonotype Responses", LaboratoryService.NavItemCategory.data, NAME, cache)); + return Collections.unmodifiableList(items); } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java index 32a0ca98..5bfee444 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java @@ -27,6 +27,7 @@ public class TCRdbSchema public static final String TABLE_MIXCR_LIBRARIES = "mixcr_libraries"; public static final String TABLE_CLONES = "clones"; + public static final String TABLE_CLONE_RESPONSES = "clone_responses"; public static final String SEQUENCE_ANALYSIS = "sequenceanalysis"; public static final String SINGLE_CELL = "singlecell"; From 477fdf67e1d15394df64240a7e08c9611d0cc44d Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 11 Jun 2025 06:47:46 -0700 Subject: [PATCH 25/34] Fix SQL syntax --- .../schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql index cd444bfe..2f92b465 100644 --- a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql @@ -16,8 +16,8 @@ ALTER TABLE tcrdb.clone_responses DROP COLUMN costim; ALTER TABLE tcrdb.clone_responses DROP COLUMN antigen; ALTER TABLE tcrdb.clone_responses DROP COLUMN stim; -ALTER TABLE tcrdb.clone_responses ADD COLUMN cdna_id int; -ALTER TABLE tcrdb.clone_responses ADD COLUMN nostimid int; -ALTER TABLE tcrdb.clone_responses ADD COLUMN chain varchar(100); -ALTER TABLE tcrdb.clone_responses ADD COLUMN clonotype varchar(1000); +ALTER TABLE tcrdb.clone_responses ADD cdna_id int; +ALTER TABLE tcrdb.clone_responses ADD nostimid int; +ALTER TABLE tcrdb.clone_responses ADD chain varchar(100); +ALTER TABLE tcrdb.clone_responses ADD clonotype varchar(1000); From 50648bff13ab843a0211609fd791c3755cfd6e54 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 12 Jun 2025 21:27:11 -0700 Subject: [PATCH 26/34] Move class --- .../labkey/sivstudies/query/SivStudiesCustomizer.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 5fba56d0..85df1e12 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -4,8 +4,10 @@ import org.labkey.api.data.AbstractTableInfo; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; +import org.labkey.api.data.SQLFragment; import org.labkey.api.data.TableInfo; import org.labkey.api.ldk.table.AbstractTableCustomizer; +import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; import org.labkey.api.security.User; import org.labkey.api.study.Dataset; @@ -41,7 +43,7 @@ public void performDatasetCustomization(DatasetTable ds) } } - private @Nullable String getStudyDemographicsSchemaTableName(Container c, User u) + private @Nullable String getStudyDemographicsSchemaTableName(Container c) { Study s = StudyService.get().getStudy(c); if (s == null) @@ -66,15 +68,14 @@ private void addAgeAtTimeCol(AbstractTableInfo ti, String dateColName) return; } - final String demographicsTableName = getStudyDemographicsSchemaTableName(ti.getUserSchema().getContainer(), ti.getUserSchema().getUser()); - final FieldKey birthFieldKey = FieldKey.fromString("Id/demographics/birth"); + final String demographicsTableName = getStudyDemographicsSchemaTableName(ti.getUserSchema().getContainer()); - //new SQLFragment("(SELECT .birth FROM studydatasets." + getStudyDemographicsSchemaTableName()) + SQLFragment sql = new SQLFragment("(SELECT t.birth FROM studydatasets." + demographicsTableName + " t WHERE t.Id = " + ExprColumn.STR_TABLE_ALIAS + ".Id)"); // TODO } - private void appendMhcColumns(AbstractTableInfo ti, String dateColName) + private void appendMhcColumns(AbstractTableInfo ti) { } From ac4773667c9dfd737e4aac44a69eae3c94e82950 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 12 Jun 2025 21:35:40 -0700 Subject: [PATCH 27/34] Missed with prior commit --- .../src/org/labkey/sivstudies}/etl/SubjectScopedSelect.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename {PMR/src/org/labkey/pmr => SivStudies/src/org/labkey/sivstudies}/etl/SubjectScopedSelect.java (99%) diff --git a/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java similarity index 99% rename from PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java rename to SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index 1c44bca3..25950512 100644 --- a/PMR/src/org/labkey/pmr/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -1,4 +1,4 @@ -package org.labkey.pmr.etl; +package org.labkey.sivstudies.etl; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; From b2e8037aa39b372d7258ef7848ae8afe89b713bb Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 13 Jun 2025 07:36:32 -0700 Subject: [PATCH 28/34] Null check --- .../src/org/labkey/sivstudies/etl/SubjectScopedSelect.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index 25950512..1cc20054 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -453,10 +453,10 @@ private List getSubjects(Logger log) } else { - Container source = ContainerManager.getForPath(_settings.get(Settings.subjectSourceContainerPath.name())); + Container source = _settings.get(Settings.subjectSourceContainerPath.name()) == null ? _containerUser.getContainer() : ContainerManager.getForPath(_settings.get(Settings.subjectSourceContainerPath.name())); if (source == null) { - source = _containerUser.getContainer(); + throw new IllegalStateException("Unknown subjectSourceContainerPath: " + _settings.get(Settings.subjectSourceContainerPath.name())); } if (!source.hasPermission(_containerUser.getUser(), ReadPermission.class)) From d353e9095c443a1dc86bd4a34b4e79324684b629 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 13 Jun 2025 09:46:08 -0700 Subject: [PATCH 29/34] Bugfixes to SubjectScopedSelect --- SivStudies/resources/etls/siv-studies.xml | 9 ++++- .../folderTypes/SIV Studies.folderType.xml | 4 +- .../study/datasets/datasets_metadata.xml | 4 +- .../sivstudies/SivStudiesController.java | 4 +- .../sivstudies/etl/SubjectScopedSelect.java | 39 +++++++++++++++---- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml index 0d1f5772..c4f1efa1 100644 --- a/SivStudies/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -13,6 +13,7 @@ + @@ -32,7 +33,8 @@ - + + @@ -53,6 +55,7 @@ + @@ -76,6 +79,7 @@ + @@ -98,7 +102,8 @@ - + + diff --git a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml index 749c8d8f..2b6cda35 100644 --- a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml +++ b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml @@ -1,5 +1,5 @@ - Studies Overview + SIV Studies Overview The default folder layout for Studies @@ -61,7 +61,7 @@ - Studies Admin + SIV Studies Admin body diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml index ecd9b819..24842c6c 100644 --- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -201,7 +201,9 @@ entityid true - + + varchar + varchar diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java index ea678411..5826667e 100644 --- a/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java @@ -66,7 +66,7 @@ public boolean handlePost(Object o, BindException errors) throws Exception StudiesService.get().importFolderDefinition(getContainer(), getUser(), ModuleLoader.getInstance().getModule(SivStudiesModule.NAME), new Path("referenceStudy")); Module m = ModuleLoader.getInstance().getModule(SivStudiesModule.NAME); - StudiesService.get().loadTsv(m.getModuleResource("data/lookup_sets.tsv"), SivStudiesSchema.NAME, getUser(), getContainer()); + StudiesService.get().loadTsv(m.getModuleResource("data/lookup_sets.tsv"), "studies", getUser(), getContainer()); Resource r = m.getModuleResource("data"); r.list().forEach(tsv -> { @@ -79,7 +79,7 @@ public boolean handlePost(Object o, BindException errors) throws Exception { case "reports.tsv" -> "laboratory"; case "species.tsv" -> "laboratory"; - default -> SivStudiesSchema.NAME; + default -> "studies"; }; StudiesService.get().loadTsv(tsv, schemaName, getUser(), getContainer()); diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index 1cc20054..56398599 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -43,6 +43,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -65,6 +66,7 @@ private enum Settings dataSourceContainerPath(false), dataSourceSchema(true), dataSourceQuery(true), + dataSourceSubjectColumn(true), dataSourceColumns(true), dataSourceColumnMapping(false), dataSourceAdditionalFilters(false), @@ -120,6 +122,11 @@ private void processBatch(List subjects, Logger log) additionalFilters.forEach(subjectFilter::addCondition); } + if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null) + { + throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name())); + } + Collection> existingRows = new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection(); if (!existingRows.isEmpty()) { @@ -132,11 +139,16 @@ private void processBatch(List subjects, Logger log) } // Query data and import - List> toImport = getRowsToImport(log); + List> toImport = getRowsToImport(subjects, log); if (!toImport.isEmpty()) { log.info("inserting " + toImport.size() + " rows"); - qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImport, new BatchValidationException(), null, null); + BatchValidationException bve = new BatchValidationException(); + qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImport, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } } else { @@ -158,7 +170,7 @@ private List parseAdditionalFilters(String rawVal) } SimpleFilter filter = new SimpleFilter(); - String[] filters = rawVal.split(","); + String[] filters = rawVal.split(";"); for (String queryParam : filters) { filter.addUrlFilters(new ActionURL().setRawQuery(queryParam), null); @@ -234,7 +246,7 @@ private Map parseColumnDefaultMap(String rawVal) return colMap; } - private List> getRowsToImport(Logger log) + private List> getRowsToImport(List subjects, Logger log) { if (_settings.get(Settings.dataSourceColumns.name()) == null) { @@ -250,6 +262,7 @@ private List> getRowsToImport(Logger log) DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.dataRemoteSource.name()), log); SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.dataSourceSchema.name()), _settings.get(Settings.dataSourceQuery.name())); sr.setColumns(sourceColumns); + sr.addFilter(_settings.get(Settings.dataSourceSubjectColumn.name()), StringUtils.join(subjects, ";"), Filter.Operator.IN); if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null) { List additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name())); @@ -269,7 +282,13 @@ else if (f.getParamVals().length == 1) value = StringUtils.join(f.getParamVals(), ";"); } - sr.addFilter(new Filter(f.getFieldKey().toString(), value, Filter.Operator.valueOf(f.getCompareType().getFilterValueText()))); + Filter.Operator o = Filter.Operator.getOperatorFromUrlKey(f.getCompareType().getPreferredUrlKey()); + if (o == null) + { + throw new IllegalStateException("Unknown operator: " + f.getCompareType().getPreferredUrlKey() + ", raw filter: " + f.getCompareType().name()); + } + + sr.addFilter(new Filter(f.getFieldKey().toString(), value, o)); } } @@ -326,15 +345,19 @@ else if (f.getParamVals().length == 1) } } + if (sourceTable.getColumn(_settings.get(Settings.dataSourceSubjectColumn.name())) == null) + { + throw new IllegalStateException("Table is missing column: " + _settings.get(Settings.dataSourceSubjectColumn.name())); + } - final SimpleFilter filter = new SimpleFilter(); + final SimpleFilter filter = new SimpleFilter(_settings.get(Settings.dataSourceSubjectColumn.name()), subjects, CompareType.IN); if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null) { List additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name())); additionalFilters.forEach(filter::addCondition); } - TableSelector ts = new TableSelector(sourceTable, PageFlowUtil.set(_settings.get(Settings.subjectSourceColumn.name())), filter, null); + TableSelector ts = new TableSelector(sourceTable, new HashSet<>(sourceColumns), filter, null); return doNameMapping(new ArrayList<>(ts.getMapCollection()), sourceToDestColumMap, columnToDefaultMap); } @@ -400,7 +423,7 @@ public List preFlightCheck(Container c) List errors = new ArrayList<>(); for (String setting : getRequiredSettings()) { - if (_settings.get(setting) == null) + if (_settings.get(setting) == null || StringUtils.isEmpty(_settings.get(setting))) { errors.add(new SimpleValidationError("Missing required setting: " + setting)); } From 989b82ab1256eece69ce91dc06ea556b85375c97 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 13 Jun 2025 10:05:15 -0700 Subject: [PATCH 30/34] Bugfixes to SubjectScopedSelect --- SivStudies/resources/etls/siv-studies.xml | 4 ++-- .../src/org/labkey/sivstudies/etl/SubjectScopedSelect.java | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml index c4f1efa1..663d8bb9 100644 --- a/SivStudies/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -34,8 +34,8 @@ - - + + diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index 56398599..fe5d05ef 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -388,6 +388,10 @@ private List> doNameMapping(List> rows, { row.put(colName, columnToDefaultMap.get(colName)); } + else if (row.get(colName) == null) + { + row.put(colName, columnToDefaultMap.get(colName)); + } } return row; From 4df642038b861860613fe6f8d5150ae022e815af Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 14 Jun 2025 12:34:51 -0700 Subject: [PATCH 31/34] Enhance calculated fields in SIV Studies --- SivStudies/resources/etls/siv-studies.xml | 24 ++- .../queries/study/demographics.query.xml | 1 + .../queries/study/demographics/.qview.xml | 19 ++ .../study/demographics/MHC Type.qview.xml | 18 ++ .../queries/study/demographicsMHC.query.xml | 31 +++ .../queries/study/demographicsMHC.sql | 29 +++ .../sivstudies/etl/SubjectScopedSelect.java | 112 +++++++--- .../query/SivStudiesCustomizer.java | 198 ++++++++++++++---- 8 files changed, 365 insertions(+), 67 deletions(-) create mode 100644 SivStudies/resources/queries/study/demographics/.qview.xml create mode 100644 SivStudies/resources/queries/study/demographics/MHC Type.qview.xml create mode 100644 SivStudies/resources/queries/study/demographicsMHC.query.xml create mode 100644 SivStudies/resources/queries/study/demographicsMHC.sql diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml index 663d8bb9..f40e3ee2 100644 --- a/SivStudies/resources/etls/siv-studies.xml +++ b/SivStudies/resources/etls/siv-studies.xml @@ -3,6 +3,28 @@ SIV_PRIMe SIV Studies / PRIMe Data + + + + + + + + + + + + + + + + + + + + + + @@ -116,6 +138,6 @@ - + diff --git a/SivStudies/resources/queries/study/demographics.query.xml b/SivStudies/resources/queries/study/demographics.query.xml index f06d781f..5c994581 100644 --- a/SivStudies/resources/queries/study/demographics.query.xml +++ b/SivStudies/resources/queries/study/demographics.query.xml @@ -6,6 +6,7 @@ + true true diff --git a/SivStudies/resources/queries/study/demographics/.qview.xml b/SivStudies/resources/queries/study/demographics/.qview.xml new file mode 100644 index 00000000..081c2872 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/.qview.xml @@ -0,0 +1,19 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml b/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml new file mode 100644 index 00000000..e0bd6d74 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsMHC.query.xml b/SivStudies/resources/queries/study/demographicsMHC.query.xml new file mode 100644 index 00000000..e07992b4 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsMHC.query.xml @@ -0,0 +1,31 @@ + + + +
+ MHC Genotype + + + true + true + + + All MHC Alleles + + + Mamu-A01 + + + Mamu-A02 + + + Mamu-B08 + + + Mamu-B17 + + + allAlleles +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsMHC.sql b/SivStudies/resources/queries/study/demographicsMHC.sql new file mode 100644 index 00000000..45f842b4 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsMHC.sql @@ -0,0 +1,29 @@ +SELECT + s.Id, + count(s.Id) as totalTests, + group_concat(DISTINCT s.assayType) as assayTypes, + + --special case A01/B17/B08 + max(CASE + WHEN (s.marker = 'Mamu-A1*001g' AND (s.result = 'POS' OR s.result = 'NEG')) THEN s.result + ELSE null + END) as A01, + + max(CASE + WHEN (s.marker = 'Mamu-A1*002g' AND (s.result = 'POS' OR s.result = 'NEG')) THEN s.result + ELSE null + END) as A02, + + max(CASE + WHEN (s.marker = 'Mamu-B*008g' AND (s.result = 'POS' OR s.result = 'NEG')) THEN s.result + ELSE '' + END) as B08, + + max(CASE + WHEN (s.marker = 'Mamu-B*017g' AND (s.result = 'POS' OR s.result = 'NEG')) THEN s.result + ELSE '' + END) as B17, + GROUP_CONCAT(distinct CASE WHEN s.result = 'POS' THEN s.marker ELSE null END, char(10)) as allAlleles +FROM study.genetics s +WHERE s.category = 'MHC Typing' +GROUP BY s.Id \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index fe5d05ef..c2589a21 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -54,6 +54,12 @@ public class SubjectScopedSelect implements TaskRefTask protected final Map _settings = new CaseInsensitiveHashMap<>(); protected ContainerUser _containerUser; + private enum MODE + { + UPDATE_ONLY, + TRUNCATE; + } + private enum Settings { subjectRemoteSource(false), @@ -90,7 +96,18 @@ public boolean isRequired() } } - final int BATCH_SIZE = 500; + final int BATCH_SIZE = 100; + + private MODE getMode() + { + String rawVal = StringUtils.trimToNull(_settings.get("mode")); + if (rawVal == null) + { + return MODE.TRUNCATE; + } + + return MODE.valueOf(rawVal); + } @Override public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException @@ -113,51 +130,90 @@ private void processBatch(List subjects, Logger log) try { - // Find / Delete existing values: - Set keyFields = destinationTable.getColumns().stream().filter(ColumnInfo::isKeyField).collect(Collectors.toSet()); - final SimpleFilter subjectFilter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name())), subjects, CompareType.IN); - if (_settings.get(Settings.targetAdditionalFilters.name()) != null) + if (getMode() == MODE.TRUNCATE) { - List additionalFilters = parseAdditionalFilters(_settings.get(Settings.targetAdditionalFilters.name())); - additionalFilters.forEach(subjectFilter::addCondition); - } + // Find / Delete existing values: + Set keyFields = destinationTable.getColumns().stream().filter(ColumnInfo::isKeyField).collect(Collectors.toSet()); + final SimpleFilter subjectFilter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name())), subjects, CompareType.IN); + if (_settings.get(Settings.targetAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.targetAdditionalFilters.name())); + additionalFilters.forEach(subjectFilter::addCondition); + } - if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null) - { - throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name())); - } + if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null) + { + throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name())); + } - Collection> existingRows = new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection(); - if (!existingRows.isEmpty()) - { - log.info("deleting " + existingRows.size() + " rows"); - qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), new ArrayList<>(existingRows), null, null); + Collection> existingRows = new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection(); + if (!existingRows.isEmpty()) + { + log.info("deleting " + existingRows.size() + " rows"); + qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), new ArrayList<>(existingRows), null, null); + } + else + { + log.info("No rows to delete for this subject batch"); + } } else { - log.info("No rows to delete for this subject batch"); + log.info("Using " + getMode().name() + " mode, source records will not be deleted"); } // Query data and import - List> toImport = getRowsToImport(subjects, log); - if (!toImport.isEmpty()) + List> toImportOrUpdate = getRowsToImport(subjects, log); + if (!toImportOrUpdate.isEmpty()) { - log.info("inserting " + toImport.size() + " rows"); - BatchValidationException bve = new BatchValidationException(); - qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImport, bve, null, null); - if (bve.hasErrors()) + if (getMode() == MODE.TRUNCATE) + { + log.info("inserting " + toImportOrUpdate.size() + " rows"); + BatchValidationException bve = new BatchValidationException(); + qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImportOrUpdate, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + else if (getMode() == MODE.UPDATE_ONLY) + { + log.info("updating " + toImportOrUpdate.size() + " rows"); + BatchValidationException bve = new BatchValidationException(); + + Collection keyFields = destinationTable.getPkColumnNames(); + List> keys = toImportOrUpdate.stream().map(x -> { + Map map = new HashMap<>(); + for (String keyField : keyFields) + { + if (x.get(keyField) != null) + { + map.put(keyField, x.get(keyField)); + } + } + + return map; + }).toList(); + + qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), toImportOrUpdate, keys, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + else { - throw bve; + throw new IllegalStateException("Unknown mode: " + getMode()); } } else { - log.info("No rows to import for this subject batch"); + log.info("No rows to import/update for this subject batch"); } } catch (SQLException | InvalidKeyException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e) { - throw new IllegalStateException("Error Importing Rows", e); + throw new IllegalStateException("Error Importing/Updating Rows", e); } } @@ -350,7 +406,7 @@ else if (f.getParamVals().length == 1) throw new IllegalStateException("Table is missing column: " + _settings.get(Settings.dataSourceSubjectColumn.name())); } - final SimpleFilter filter = new SimpleFilter(_settings.get(Settings.dataSourceSubjectColumn.name()), subjects, CompareType.IN); + final SimpleFilter filter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.dataSourceSubjectColumn.name())), subjects, CompareType.IN); if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null) { List additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name())); diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 85df1e12..b1b4de83 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -1,22 +1,31 @@ package org.labkey.sivstudies.query; -import org.jetbrains.annotations.Nullable; +import org.apache.logging.log4j.Logger; import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; -import org.labkey.api.data.SQLFragment; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.WrappedColumn; import org.labkey.api.ldk.table.AbstractTableCustomizer; -import org.labkey.api.query.ExprColumn; -import org.labkey.api.query.FieldKey; +import org.labkey.api.query.LookupForeignKey; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryForeignKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; import org.labkey.api.security.User; -import org.labkey.api.study.Dataset; import org.labkey.api.study.DatasetTable; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; +import org.labkey.api.util.logging.LogHelper; + +import java.util.ArrayList; +import java.util.List; public class SivStudiesCustomizer extends AbstractTableCustomizer { + public static final String ID_COL = "Id"; + private static final Logger _log = LogHelper.getLogger(SivStudiesCustomizer.class, "Table customization for the SIV Studies module"); + @Override public void customize(TableInfo tableInfo) { @@ -28,56 +37,158 @@ public void customize(TableInfo tableInfo) public void performDatasetCustomization(DatasetTable ds) { - _log.debug("Customizing: {}", ds.getName()); - - if (!ds.getDataset().isDemographicData()) + if (ds instanceof AbstractTableInfo ati) { - if (ds instanceof AbstractTableInfo ati) + _log.debug("Customizing dataset: {}", ds.getName()); + + if (!ds.getDataset().isDemographicData()) { - addAgeAtTimeCol(ati, "date"); + appendAgeAtTimeCol(ds.getUserSchema(), ati, "date"); } - else + + if ("demographics".equalsIgnoreCase(ds.getName())) { - _log.error("Expected DatasetTable to be instanceof AbstractTableInfo"); + appendDemographicsColumns(ati); } } - } - - private @Nullable String getStudyDemographicsSchemaTableName(Container c) - { - Study s = StudyService.get().getStudy(c); - if (s == null) + else { - return null; - } - - Dataset ds = s.getDatasetByLabel("demographics"); - if (ds == null) - { - return null; + _log.error("Expected DatasetTable to be instanceof AbstractTableInfo: " + ds.getName()); } + } - return ds.getDomain().getStorageTableName(); + private ColumnInfo getPkCol(TableInfo ti) + { + List pks = ti.getPkColumns(); + return (pks.size() != 1) ? null : pks.get(0); } - private void addAgeAtTimeCol(AbstractTableInfo ti, String dateColName) + private void appendAgeAtTimeCol(UserSchema demographicsSchema, AbstractTableInfo ds, final String dateColName) { - final String name = "ageAtTime"; - if (ti.getColumn(name, false) != null) - { + String name = "ageAtTime"; + if (ds.getColumn(name, false) != null) + return; + + final ColumnInfo pkCol = getPkCol(ds); + if (pkCol == null) return; - } - final String demographicsTableName = getStudyDemographicsSchemaTableName(ti.getUserSchema().getContainer()); + final ColumnInfo idCol = ds.getColumn(ID_COL); + if (idCol == null) + return; + + if (ds.getColumn(dateColName) == null) + return; - SQLFragment sql = new SQLFragment("(SELECT t.birth FROM studydatasets." + demographicsTableName + " t WHERE t.Id = " + ExprColumn.STR_TABLE_ALIAS + ".Id)"); + final String targetSchemaName = ds.getUserSchema().getName(); + final Container targetSchemaContainer = ds.getUserSchema().getContainer(); + final User u = ds.getUserSchema().getUser(); + final String schemaName = ds.getPublicSchemaName(); + final String queryName = ds.getName(); + final String demographicsPath = demographicsSchema.getContainer().getPath(); + + WrappedColumn col = new WrappedColumn(pkCol, name); + col.setLabel("Age At The Time"); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + col.setFk(new LookupForeignKey(){ + @Override + public TableInfo getLookupTableInfo() + { + String name = queryName + "_ageAtTime"; + UserSchema targetSchema = ds.getUserSchema().getDefaultSchema().getUserSchema(targetSchemaName); + QueryDefinition qd = QueryService.get().createQueryDef(u, targetSchemaContainer, targetSchema, name); + qd.setSql("SELECT\n" + + "c." + pkCol.getFieldKey().toSQLString() + ",\n" + + "\n" + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " ROUND(CONVERT(age_in_months(d.birth, d.death), DOUBLE) / 12, 1)\n" + + "ELSE\n" + + " ROUND(CONVERT(age_in_months(d.birth, CAST(c." + dateColName + " as DATE)), DOUBLE) / 12, 1)\n" + + "END AS float) as AgeAtTime,\n" + + "\n" + - // TODO + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " ROUND(CONVERT(timestampdiff('SQL_TSI_DAY', d.birth, d.death), DOUBLE) / 365.25, 2)\n" + + "ELSE\n" + + " ROUND(CONVERT(timestampdiff('SQL_TSI_DAY', d.birth, CAST(c." + dateColName + " as DATE)), DOUBLE) / 365.25, 2)\n" + + "END AS float) as AgeAtTimeYears,\n" + + "\n" + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " floor(age(d.birth, d.death))\n" + + "ELSE\n" + + " floor(age(d.birth, CAST(c." + dateColName + " as DATE)))\n" + + "END AS float) as AgeAtTimeYearsRounded,\n" + + "\n" + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY',d.birth, d.death), INTEGER)\n" + + "ELSE\n" + + " CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY',d.birth, CAST(c." + dateColName + " AS DATE)), INTEGER)\n" + + "END AS float) as AgeAtTimeDays,\n" + + "\n" + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " CONVERT(age_in_months(d.birth, d.death), INTEGER)\n" + + "ELSE\n" + + " CONVERT(age_in_months(d.birth, CAST(c." + dateColName + " AS DATE)), INTEGER)\n" + + "END AS float) as AgeAtTimeMonths,\n" + + "FROM \"" + schemaName + "\".\"" + queryName + "\" c " + + "LEFT JOIN \"" + demographicsPath + "\".study.demographics d ON (d.Id = c." + idCol.getFieldKey().toSQLString() + ")" + ); + qd.setIsTemporary(true); + + List errors = new ArrayList<>(); + TableInfo ti = qd.getTable(errors, true); + if (!errors.isEmpty()) + { + _log.warn("Error creating age at time lookup table for: " + schemaName + "." + queryName + " in container: " + targetSchema.getContainer().getPath()); + for (QueryException e : errors) + { + _log.warn(e.getMessage(), e); + } + } + + if (ti != null) + { + ((BaseColumnInfo)ti.getColumn(pkCol.getName())).setHidden(true); + ((BaseColumnInfo)ti.getColumn(pkCol.getName())).setKeyField(true); + } + + return ti; + } + }); + + ds.addColumn(col); } - private void appendMhcColumns(AbstractTableInfo ti) + private void appendDemographicsColumns(AbstractTableInfo demographicsTable) { - + if (demographicsTable.getColumn("mhcGenotypes") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(demographicsTable.getUserSchema(), "demographicsMHC", demographicsTable, "mhcGenotypes"); + colInfo.setLabel("MHC Genotypes"); + demographicsTable.addColumn(colInfo); + } } private void appendPvlColumns(AbstractTableInfo ti, ColumnInfo subjectCol, ColumnInfo dateCol) @@ -97,4 +208,15 @@ private void appendPvlColumns(AbstractTableInfo ti, ColumnInfo subjectCol, Colum // ti.addColumn(newCol); // } } + + private BaseColumnInfo getWrappedIdCol(UserSchema targetQueryUserSchema, String targetQueryName, AbstractTableInfo demographicsTable, String colName) + { + WrappedColumn col = new WrappedColumn(demographicsTable.getColumn(ID_COL), colName); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + col.setFk(new QueryForeignKey(demographicsTable.getUserSchema(), null, targetQueryUserSchema, null, targetQueryName, ID_COL, ID_COL)); + + return col; + } } From 57f79ff747edddb72fa41cbf3013f4e0a53a2e74 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 16 Jun 2025 07:29:43 -0700 Subject: [PATCH 32/34] Update TCRdb schema --- .../postgresql/tcrdb-15.53-15.54.sql | 20 ++++++ .../dbscripts/sqlserver/tcrdb-15.53-15.54.sql | 20 ++++++ tcrdb/resources/schemas/tcrdb.xml | 66 +++++++++++++++++++ tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 2 +- tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java | 1 + tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java | 1 + 6 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.53-15.54.sql create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.53-15.54.sql diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.53-15.54.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.53-15.54.sql new file mode 100644 index 00000000..743dffcb --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.53-15.54.sql @@ -0,0 +1,20 @@ +CREATE TABLE tcrdb.stims ( + rowid SERIAL, + cdna_id int, + controlStimId int, + + quantificationMethod varchar(1000), + quantification double precision, + + flowQuantificationMethod varchar(1000), + flowQuantification double precision, + comment varchar(4000), + + container entityid, + created timestamp, + createdby int, + modified timestamp, + modifiedby int, + + constraint PK_stims PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.53-15.54.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.53-15.54.sql new file mode 100644 index 00000000..54362b3c --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.53-15.54.sql @@ -0,0 +1,20 @@ +CREATE TABLE tcrdb.stims ( + rowid int identity(1,1), + cdna_id int, + controlStimId int, + + quantificationMethod varchar(1000), + quantification double precision, + + flowQuantificationMethod varchar(1000), + flowQuantification double precision, + comment varchar(4000), + + container entityid, + created datetime, + createdby int, + modified datetime, + modifiedby int, + + constraint PK_stims PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index 027dc8c3..88951dd5 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -258,4 +258,70 @@ + + + + Stim Experiments + rowid + + + Row Id + true + + + cDNA ID + + singlecell + cdna_libraries + rowid + + + + No Stim / Control cDNA ID + + singlecell + cdna_libraries + rowid + + + + Quantification Method + + + Quantification + + + Flow Quantification Method + + + Flow Quantification + + + + + + true + + + true + + + false + false + false + true + true + + + true + + + false + false + false + true + true + + +
\ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index c6ec96f3..dd7b9d91 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -46,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.53; + return 15.54; } @Override diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java index 8f460121..27895a1c 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java @@ -70,6 +70,7 @@ public List getDataNavItems(Container c, User u) items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, "TCR Clones", LaboratoryService.NavItemCategory.data, NAME, cache)); items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONE_RESPONSES, "TCR Clonotype Responses", LaboratoryService.NavItemCategory.data, NAME, cache)); + items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIM_EXPERIMENTS, "TCR Clonotype Responses", LaboratoryService.NavItemCategory.data, NAME, cache)); return Collections.unmodifiableList(items); } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java index 5bfee444..5bafb8f0 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java @@ -28,6 +28,7 @@ public class TCRdbSchema public static final String TABLE_MIXCR_LIBRARIES = "mixcr_libraries"; public static final String TABLE_CLONES = "clones"; public static final String TABLE_CLONE_RESPONSES = "clone_responses"; + public static final String TABLE_STIM_EXPERIMENTS = "stims"; public static final String SEQUENCE_ANALYSIS = "sequenceanalysis"; public static final String SINGLE_CELL = "singlecell"; From ee4c1c10d0db0c11afad2abe10b244c02311840e Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 16 Jun 2025 08:14:59 -0700 Subject: [PATCH 33/34] Update TCRdb views --- .../resources/queries/tcrdb/stims/.qview.xml | 19 +++++++++++++++++++ tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tcrdb/resources/queries/tcrdb/stims/.qview.xml diff --git a/tcrdb/resources/queries/tcrdb/stims/.qview.xml b/tcrdb/resources/queries/tcrdb/stims/.qview.xml new file mode 100644 index 00000000..0aebcb9a --- /dev/null +++ b/tcrdb/resources/queries/tcrdb/stims/.qview.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java index 27895a1c..6af64dd6 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java @@ -70,7 +70,7 @@ public List getDataNavItems(Container c, User u) items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, "TCR Clones", LaboratoryService.NavItemCategory.data, NAME, cache)); items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONE_RESPONSES, "TCR Clonotype Responses", LaboratoryService.NavItemCategory.data, NAME, cache)); - items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIM_EXPERIMENTS, "TCR Clonotype Responses", LaboratoryService.NavItemCategory.data, NAME, cache)); + items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIM_EXPERIMENTS, "T-cell Stim Experiments", LaboratoryService.NavItemCategory.data, NAME, cache)); return Collections.unmodifiableList(items); } From e0e8d67a07627903b36fe3ab6599a420e0c34f55 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 16 Jun 2025 09:00:47 -0700 Subject: [PATCH 34/34] Update TCRdb views --- tcrdb/resources/queries/tcrdb/stims/.qview.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tcrdb/resources/queries/tcrdb/stims/.qview.xml b/tcrdb/resources/queries/tcrdb/stims/.qview.xml index 0aebcb9a..65170b8d 100644 --- a/tcrdb/resources/queries/tcrdb/stims/.qview.xml +++ b/tcrdb/resources/queries/tcrdb/stims/.qview.xml @@ -1,10 +1,11 @@ - - - - + + + + +