diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml index 1186b27a4..a0fc89c53 100644 --- a/PMR/resources/etls/pmr-datasets.xml +++ b/PMR/resources/etls/pmr-datasets.xml @@ -189,8 +189,7 @@ - - + diff --git a/SivStudies/build.gradle b/SivStudies/build.gradle new file mode 100644 index 000000000..68d68d563 --- /dev/null +++ b/SivStudies/build.gradle @@ -0,0 +1,20 @@ +import org.labkey.gradle.util.BuildUtils + +plugins { + id 'org.labkey.build.module' +} + +dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:premiumModules:dataintegration", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module") + + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:premiumModules:dataintegration", depProjectConfig: "apiJarFile") + + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") +} diff --git a/SivStudies/module.properties b/SivStudies/module.properties new file mode 100644 index 000000000..142ebb1a1 --- /dev/null +++ b/SivStudies/module.properties @@ -0,0 +1,7 @@ +ModuleClass: org.labkey.sivstudies.SivStudiesModule +Label: SIV Studies +Description: A module designed to manage data from SIV and related studies. +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: false +SupportedDatabases: mssql, pgsql diff --git a/SivStudies/resources/data/amount_units.tsv b/SivStudies/resources/data/amount_units.tsv new file mode 100644 index 000000000..b4b010264 --- /dev/null +++ b/SivStudies/resources/data/amount_units.tsv @@ -0,0 +1,11 @@ +value +g +IU +U +mEq +mg +no units +ug +units +L/min +% \ No newline at end of file diff --git a/SivStudies/resources/data/calculated_status_codes.tsv b/SivStudies/resources/data/calculated_status_codes.tsv new file mode 100644 index 000000000..d2bf1dec4 --- /dev/null +++ b/SivStudies/resources/data/calculated_status_codes.tsv @@ -0,0 +1,6 @@ +value +Alive +Dead +No Record +Shipped +Unknown \ No newline at end of file diff --git a/SivStudies/resources/data/conc_units.tsv b/SivStudies/resources/data/conc_units.tsv new file mode 100644 index 000000000..d7d9b4d18 --- /dev/null +++ b/SivStudies/resources/data/conc_units.tsv @@ -0,0 +1,12 @@ +value denominator numerator +g/ml mL g +g/tsp tsp g +IU/ml mL IU +mEq/ml mL mEq +mg/capsule capsule(s) mg +mg/ml mL mg +mg/piece piece(s) mg +mg/tablet tablet(s) mg +mg/tsp tsp mg +ug/ml mL ug +units/ml mL units \ No newline at end of file diff --git a/SivStudies/resources/data/dosage_units.tsv b/SivStudies/resources/data/dosage_units.tsv new file mode 100644 index 000000000..eb8de1491 --- /dev/null +++ b/SivStudies/resources/data/dosage_units.tsv @@ -0,0 +1,11 @@ +value numerator denominator +g/kg g kg +IU/kg IU kg +mEq/kg mEq kg +mg/animal mg +mg/kg mg kg +ml/kg ml kg +no units +ounces/kg ounces kg +ug/kg ug kg +units/kg units kg \ No newline at end of file diff --git a/SivStudies/resources/data/gender_codes.tsv b/SivStudies/resources/data/gender_codes.tsv new file mode 100644 index 000000000..07b681950 --- /dev/null +++ b/SivStudies/resources/data/gender_codes.tsv @@ -0,0 +1,4 @@ +v meaning origgender +f Female f +m Male m +u Unknown \ No newline at end of file diff --git a/SivStudies/resources/data/geographic_origins.tsv b/SivStudies/resources/data/geographic_origins.tsv new file mode 100644 index 000000000..0870abf80 --- /dev/null +++ b/SivStudies/resources/data/geographic_origins.tsv @@ -0,0 +1,4 @@ +value +Indian +Chinese +Hybrid \ No newline at end of file diff --git a/SivStudies/resources/data/lookup_sets.tsv b/SivStudies/resources/data/lookup_sets.tsv new file mode 100644 index 000000000..8e2b94822 --- /dev/null +++ b/SivStudies/resources/data/lookup_sets.tsv @@ -0,0 +1,9 @@ +setname label keyfield titleColumn +amount_units Amount Units unit +calculated_status_codes Calculated Status Codes code +conc_units Concentraiton Units unit +dosage_units Dosage Units unit +gender_codes Gender Codes +geographic_origins Geographic Origins origin +routes Routes route +volume_units Volume Units unit \ No newline at end of file diff --git a/SivStudies/resources/data/reports.tsv b/SivStudies/resources/data/reports.tsv new file mode 100644 index 000000000..4dd48af51 --- /dev/null +++ b/SivStudies/resources/data/reports.tsv @@ -0,0 +1,12 @@ +reportname category reporttype reporttitle visible containerpath schemaname queryname viewname report datefieldname todayonly queryhaslocation sort_order QCStateLabelFieldName description +activeAssignments Assignments and Groups query Active Assignments true study Assignment Active Assignments date false false qcstate/publicdata This report shows the active assignments for each animal +assignmentHistory Assignments and Groups query Assignment History true study Assignment date false false qcstate/publicdata This report shows all assignments records for the animals +activeGroups Assignments and Groups query Active Groups true study animal_group_members Active Members date false false qcstate/publicdata This report shows the active assignments for each animal +groupHistory Assignments and Groups query Group History true study animal_group_members date false false qcstate/publicdata This report shows all assignments records for the animals +microbiology Lab Results query Microbiology true study Microbiology Results date false false qcstate/publicdata +biochemistry Lab Results js Biochemistry true study bloodChemistry date false false Contains results of chemistry panels. Can be displayed either by panel, or showing reference ranges +clinPathRuns Lab Results query Lab Runs true study Clinpath Runs date false false qcstate/publicdata Contains all clinpath requests +iStat Lab Results js iStat true study iStat date false false qcstate/publicdata Contains iStat results +hematology Lab Results js Hematology true study hematology date false false Contains hematology data showing cell subsets +parasitology Lab Results query Parasitology true study Parasitology Results date false false qcstate/publicdata Contains results of parasitology testing +urinalysis Lab Results js Urinalysis true study urinalysisResults date false false Contains urinalysis results diff --git a/SivStudies/resources/data/routes.tsv b/SivStudies/resources/data/routes.tsv new file mode 100644 index 000000000..b0e1f0d80 --- /dev/null +++ b/SivStudies/resources/data/routes.tsv @@ -0,0 +1,24 @@ +value title +IM +intracardiac +intracarotid +intracorneal Intracorneal +intracranial +IP intraperitoneal +ID +INH +IT +IV +CRI +IVAG +oral +PO +rectal +Spillage +SQ +OU +OD +OS +topical +topical (eye) +topical (skin) \ No newline at end of file diff --git a/SivStudies/resources/data/species.tsv b/SivStudies/resources/data/species.tsv new file mode 100644 index 000000000..0e063c4cf --- /dev/null +++ b/SivStudies/resources/data/species.tsv @@ -0,0 +1,10 @@ +common scientific_name id_prefix mhc_prefix blood_per_kg max_draw_pct blood_draw_interval cites_code dateDisabled +Baboon 60 0.2 30 +Cotton-top Tamarin Saguinus oedipus so Saoe 60 0.2 30 +Cynomolgus Macaca fascicularis cy Mafa 60 0.2 30 +Marmoset Callithrix jacchus cj Caja 60 0.15 30 +Pigtail Macaca Nemestrina Mane 60 0.2 30 +Rhesus Macaca mulatta r|rh Mamu 60 0.2 30 +Sooty Mangabey Cercocebus atys Ceat 60 0.2 30 +Stump Tailed Macaca Arctoides Maar 60 0.2 30 +Vervet Chlorocebus sabaeus ag Chsa 60 0.2 30 diff --git a/SivStudies/resources/data/volume_units.tsv b/SivStudies/resources/data/volume_units.tsv new file mode 100644 index 000000000..fc2c76822 --- /dev/null +++ b/SivStudies/resources/data/volume_units.tsv @@ -0,0 +1,13 @@ +value +capsule(s) +cup +drop(s) +cube +mL +mL/hr +no units +ounce(s) +pan +piece(s) +tablet(s) +tsp \ No newline at end of file diff --git a/SivStudies/resources/etls/siv-studies.xml b/SivStudies/resources/etls/siv-studies.xml new file mode 100644 index 000000000..f40e3ee27 --- /dev/null +++ b/SivStudies/resources/etls/siv-studies.xml @@ -0,0 +1,143 @@ + + + SIV_PRIMe + SIV Studies / PRIMe Data + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SivStudies/resources/folderTypes/SIV Studies.folderType.xml b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml new file mode 100644 index 000000000..2b6cda35a --- /dev/null +++ b/SivStudies/resources/folderTypes/SIV Studies.folderType.xml @@ -0,0 +1,78 @@ + + SIV Studies Overview + The default folder layout for Studies + + + + + + + + + + + + + + + + + datasets + Datasets + + + datasets + + + + + + + Datasets + body + + + + + + + + + + + + + + + + + + + + + + admin + Admin + + + + + + + + + SIV Studies Admin + body + + + + + + + + + studies + + studies + true + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml new file mode 100644 index 000000000..05d40e8ca --- /dev/null +++ b/SivStudies/resources/queries/study/assignment.query.xml @@ -0,0 +1,31 @@ + + + + + + + + Date Added + Date + + + End Date + Date + + + Study + + + Sub-Group + + + Cohort ID + + + Category + + +
+ + + diff --git a/SivStudies/resources/queries/study/demographics.query.xml b/SivStudies/resources/queries/study/demographics.query.xml new file mode 100644 index 000000000..5c9945815 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics.query.xml @@ -0,0 +1,66 @@ + + + + + + + + + true + + + true + false + + + Date + Birth + /query/executeQuery.view? + schemaName=study& + query.queryName=Birth& + query.Id~eq=${Id} + + + + Date + Death + /query/executeQuery.view? + schemaName=study& + query.queryName=Deaths& + query.Id~eq=${Id} + + + + true + false + true + + + Sex + + + Geographic Origin + + + Birth + + + Death + + + Species + + + Mother + + + Father + + + Status + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographics/.qview.xml b/SivStudies/resources/queries/study/demographics/.qview.xml new file mode 100644 index 000000000..081c28720 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/.qview.xml @@ -0,0 +1,19 @@ + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml b/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml new file mode 100644 index 000000000..e0bd6d743 --- /dev/null +++ b/SivStudies/resources/queries/study/demographics/MHC Type.qview.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/demographicsMHC.query.xml b/SivStudies/resources/queries/study/demographicsMHC.query.xml new file mode 100644 index 000000000..e07992b49 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsMHC.query.xml @@ -0,0 +1,31 @@ + + + + + MHC Genotype + + + true + true + + + All MHC Alleles + + + Mamu-A01 + + + Mamu-A02 + + + Mamu-B08 + + + Mamu-B17 + + + allAlleles +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsMHC.sql b/SivStudies/resources/queries/study/demographicsMHC.sql new file mode 100644 index 000000000..45f842b4a --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsMHC.sql @@ -0,0 +1,29 @@ +SELECT + s.Id, + count(s.Id) as totalTests, + group_concat(DISTINCT s.assayType) as assayTypes, + + --special case A01/B17/B08 + max(CASE + WHEN (s.marker = 'Mamu-A1*001g' AND (s.result = 'POS' OR s.result = 'NEG')) THEN s.result + ELSE null + END) as A01, + + max(CASE + WHEN (s.marker = 'Mamu-A1*002g' AND (s.result = 'POS' OR s.result = 'NEG')) THEN s.result + ELSE null + END) as A02, + + max(CASE + WHEN (s.marker = 'Mamu-B*008g' AND (s.result = 'POS' OR s.result = 'NEG')) THEN s.result + ELSE '' + END) as B08, + + max(CASE + WHEN (s.marker = 'Mamu-B*017g' AND (s.result = 'POS' OR s.result = 'NEG')) THEN s.result + ELSE '' + END) as B17, + GROUP_CONCAT(distinct CASE WHEN s.result = 'POS' THEN s.marker ELSE null END, char(10)) as allAlleles +FROM study.genetics s +WHERE s.category = 'MHC Typing' +GROUP BY s.Id \ No newline at end of file diff --git a/SivStudies/resources/queries/study/flags.query.xml b/SivStudies/resources/queries/study/flags.query.xml new file mode 100644 index 000000000..6e2b23eda --- /dev/null +++ b/SivStudies/resources/queries/study/flags.query.xml @@ -0,0 +1,22 @@ + + + + + + + + Date Added + Date + + + End Date + Date + + + Flag + + +
+ + + diff --git a/SivStudies/resources/queries/study/genetics.query.xml b/SivStudies/resources/queries/study/genetics.query.xml new file mode 100644 index 000000000..54b08daef --- /dev/null +++ b/SivStudies/resources/queries/study/genetics.query.xml @@ -0,0 +1,30 @@ + + + + + + + + Date Added + Date + + + Category + + + Assay Type + + + Marker/Allele + + + Result + + + Score + + +
+ + + diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml new file mode 100644 index 000000000..833d0fde2 --- /dev/null +++ b/SivStudies/resources/queries/study/immunizations.query.xml @@ -0,0 +1,33 @@ + + + + + + + + Date Added + Date + + + Category + + + Treatment + + + Route + + + Quantity + + + Quantity Units + + + Reason + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/labwork.query.xml b/SivStudies/resources/queries/study/labwork.query.xml new file mode 100644 index 000000000..99e85a14a --- /dev/null +++ b/SivStudies/resources/queries/study/labwork.query.xml @@ -0,0 +1,35 @@ + + + + + + + + Date + Date + + + Category + + + Test + + + Result + + + Units + + + Qualitative Result + true + false + + + Method + + +
+ + + diff --git a/SivStudies/resources/queries/study/procedures.query.xml b/SivStudies/resources/queries/study/procedures.query.xml new file mode 100644 index 000000000..b866f8b5f --- /dev/null +++ b/SivStudies/resources/queries/study/procedures.query.xml @@ -0,0 +1,22 @@ + + + + + + + + + + true + + + Category + + + Procedure + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml new file mode 100644 index 000000000..87ac54943 --- /dev/null +++ b/SivStudies/resources/queries/study/samples.query.xml @@ -0,0 +1,31 @@ + + + + + + + + Date Added + Date + + + Sample ID + + + Sample Type + + + Preservation + + + Quantity + + + Quantity Units + + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml new file mode 100644 index 000000000..bf1d995a8 --- /dev/null +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + Date + + + End Date + + + Key + true + false + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/queries/study/treatments.query.xml b/SivStudies/resources/queries/study/treatments.query.xml new file mode 100644 index 000000000..1a5e82c98 --- /dev/null +++ b/SivStudies/resources/queries/study/treatments.query.xml @@ -0,0 +1,72 @@ + + + + + + + + Date + Date + + + End Date + Date + + + Category + + + Treatment + + + Route + + + Frequency + + + Amount + + + Amount Units + + + Volume + true + false + + + Volume Units + true + false + + + Concentration + true + false + + + Conc. Units + true + false + + + Dosage + true + false + + + Dosage Units + true + false + + + Reason + true + false + + +
+ + + diff --git a/SivStudies/resources/queries/study/viralloads.query.xml b/SivStudies/resources/queries/study/viralloads.query.xml new file mode 100644 index 000000000..ef00f0e25 --- /dev/null +++ b/SivStudies/resources/queries/study/viralloads.query.xml @@ -0,0 +1,42 @@ + + + + + + + + Date + Date + + + Sample Type + + + Assay Type + + + target + + + LOD + + + Result + + + Units + + + Qualitative Result + true + false + + + true + false + + +
+ + + diff --git a/SivStudies/resources/queries/study/weight.query.xml b/SivStudies/resources/queries/study/weight.query.xml new file mode 100644 index 000000000..24e484dd7 --- /dev/null +++ b/SivStudies/resources/queries/study/weight.query.xml @@ -0,0 +1,20 @@ + + + + + + + + + + true + + + Weight (kg) + 0.#### + + +
+ + + \ No newline at end of file diff --git a/SivStudies/resources/referenceStudy/folder.xml b/SivStudies/resources/referenceStudy/folder.xml new file mode 100644 index 000000000..e3acbb155 --- /dev/null +++ b/SivStudies/resources/referenceStudy/folder.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/SivStudies/resources/referenceStudy/study/datasets/Studies.dataset b/SivStudies/resources/referenceStudy/study/datasets/Studies.dataset new file mode 100644 index 000000000..8e5970288 --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/datasets/Studies.dataset @@ -0,0 +1,19 @@ +# default group can be used to avoid repeating definitions for each dataset +# +# action=[REPLACE,APPEND,DELETE] (default:REPLACE) +# deleteAfterImport=[TRUE|FALSE] (default:FALSE) + +default.action=REPLACE +default.deleteAfterImport=FALSE + +# map a source tsv column (right side) to a property name or full propertyURI (left) +# predefined properties: ParticipantId, SiteId, VisitId, Created +default.property.ParticipantId=ptid +default.property.Created=dfcreate + +# use to map from filename->datasetid +# NOTE: if there are NO explicit import definitions, we will try to import all files matching pattern +# NOTE: if there are ANY explicit mapping, we will only import listed datasets + +default.filePattern=dataset(\\d*).tsv +default.importAllMatches=TRUE diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml new file mode 100644 index 000000000..346bd0300 --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_manifest.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml new file mode 100644 index 000000000..24842c6ca --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -0,0 +1,394 @@ + + + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + timestamp + + + varchar + + + Flags/Misc Information +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + timestamp + + + varchar + + + varchar + + + double + + + varchar + + + double + + + varchar + + + double + + + varchar + + + varchar + + + integer + + + double + + + varchar + + + varchar + + + Medications/Treatments +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + varchar + + + varchar + + + varchar + + + double + + + varchar + + + varchar + + + Immunizations +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + double + + + Weight +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + varchar + + + varchar + + + varchar + + + double + + + double + + + varchar + true + + org.labkey.studies.query.ResultsOOODisplayColumn + + + + varchar + + + varchar + + + Viral Loads +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + varchar + + + varchar + + + double + + + varchar + + + varchar + + + varchar + + + varchar + + + Lab Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + false + + + + varchar + + + varchar + + + timestamp + + + timestamp + + + varchar + + + varchar + + + varchar + + + varchar + + + Demographics +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + timestamp + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + Project Assignment +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + + varchar + + + varchar + + + varchar + + + double + + + varchar + + + Samples +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + varchar + + + varchar + + + varchar + + + varchar + + + double + + + Genetic Data +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + + timestamp + http://cpas.labkey.com/laboratory#sampleDate + + + entityid + true + + + varchar + + + varchar + + + Procedures +
+ diff --git a/SivStudies/resources/referenceStudy/study/study.xml b/SivStudies/resources/referenceStudy/study/study.xml new file mode 100644 index 000000000..0acc6d7dc --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/study.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/SivStudies/resources/referenceStudy/study/studyPolicy.xml b/SivStudies/resources/referenceStudy/study/studyPolicy.xml new file mode 100644 index 000000000..3755d25b8 --- /dev/null +++ b/SivStudies/resources/referenceStudy/study/studyPolicy.xml @@ -0,0 +1,10 @@ + + + BASIC_WRITE + + + + + + + \ No newline at end of file diff --git a/SivStudies/resources/schemas/dbscripts/postgresql/sivstudies-25.000-25.001.sql b/SivStudies/resources/schemas/dbscripts/postgresql/sivstudies-25.000-25.001.sql new file mode 100644 index 000000000..cbf0c4f48 --- /dev/null +++ b/SivStudies/resources/schemas/dbscripts/postgresql/sivstudies-25.000-25.001.sql @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for SivStudies module here +-- All SQL VIEW definitions should be created in sivstudies-create.sql and dropped in sivstudies-drop.sql +CREATE SCHEMA sivstudies; diff --git a/SivStudies/resources/schemas/sivstudies.xml b/SivStudies/resources/schemas/sivstudies.xml new file mode 100644 index 000000000..f2708fff2 --- /dev/null +++ b/SivStudies/resources/schemas/sivstudies.xml @@ -0,0 +1,2 @@ + \ No newline at end of file diff --git a/SivStudies/resources/views/studiesAdmin.html b/SivStudies/resources/views/studiesAdmin.html new file mode 100644 index 000000000..b17398a9c --- /dev/null +++ b/SivStudies/resources/views/studiesAdmin.html @@ -0,0 +1,33 @@ + \ No newline at end of file diff --git a/SivStudies/resources/views/studiesAdmin.view.xml b/SivStudies/resources/views/studiesAdmin.view.xml new file mode 100644 index 000000000..198818fc9 --- /dev/null +++ b/SivStudies/resources/views/studiesAdmin.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/SivStudies/resources/views/studiesAdmin.webpart.xml b/SivStudies/resources/views/studiesAdmin.webpart.xml new file mode 100644 index 000000000..21f7c9a68 --- /dev/null +++ b/SivStudies/resources/views/studiesAdmin.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java new file mode 100644 index 000000000..5826667e0 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesController.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.sivstudies; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.resource.Resource; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.studies.StudiesService; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.HtmlView; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.servlet.ModelAndView; + +public class SivStudiesController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SivStudiesController.class); + public static final String NAME = "sivstudies"; + + private static final Logger _log = LogHelper.getLogger(SivStudiesController.class, "SivStudiesController messages"); + + public SivStudiesController() + { + setActionResolver(_actionResolver); + } + + @RequiresPermission(AdminPermission.class) + public static class ImportStudyAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + setTitle("Import Study"); + + return new HtmlView(HtmlString.unsafe("This will import the default study in this folder, and truncate/load ancillary data. Do you want to continue?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + StudiesService.get().importFolderDefinition(getContainer(), getUser(), ModuleLoader.getInstance().getModule(SivStudiesModule.NAME), new Path("referenceStudy")); + + Module m = ModuleLoader.getInstance().getModule(SivStudiesModule.NAME); + StudiesService.get().loadTsv(m.getModuleResource("data/lookup_sets.tsv"), "studies", getUser(), getContainer()); + + Resource r = m.getModuleResource("data"); + r.list().forEach(tsv -> { + if ("lookup_sets.tsv".equals(tsv.getName())) + { + return; + } + + String schemaName = switch (tsv.getName()) + { + case "reports.tsv" -> "laboratory"; + case "species.tsv" -> "laboratory"; + default -> "studies"; + }; + + StudiesService.get().loadTsv(tsv, schemaName, getUser(), getContainer()); + }); + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + + } + + @NotNull + @Override + public URLHelper getSuccessURL(Object o) + { + return PageFlowUtil.urlProvider(PipelineUrls.class).urlBegin(getContainer()); + } + } +} diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesManager.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesManager.java new file mode 100644 index 000000000..9682dadb2 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesManager.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.sivstudies; + +public class SivStudiesManager +{ + private static final SivStudiesManager _instance = new SivStudiesManager(); + + private SivStudiesManager() + { + // prevent external construction with a private default constructor + } + + public static SivStudiesManager get() + { + return _instance; + } +} \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java new file mode 100644 index 000000000..020e98897 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.sivstudies; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.ldk.ExtendedSimpleModule; +import org.labkey.api.module.ModuleContext; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +public class SivStudiesModule extends ExtendedSimpleModule +{ + public static final String NAME = "SivStudies"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 25.001; + } + + @Override + protected void init() + { + addController(SivStudiesController.NAME, SivStudiesController.class); + } + + @Override + public void doStartupAfterSpringConfig(ModuleContext moduleContext) + { + + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return Collections.singleton(SivStudiesSchema.NAME); + } +} \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesSchema.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesSchema.java new file mode 100644 index 000000000..608fbdf15 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesSchema.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.sivstudies; + +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.dialect.SqlDialect; + +public class SivStudiesSchema +{ + private static final SivStudiesSchema _instance = new SivStudiesSchema(); + public static final String NAME = "sivstudies"; + + public static SivStudiesSchema getInstance() + { + return _instance; + } + + private SivStudiesSchema() + { + + } + + public DbSchema getSchema() + { + return DbSchema.get(NAME, DbSchemaType.Module); + } + + public SqlDialect getSqlDialect() + { + return getSchema().getSqlDialect(); + } +} diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java new file mode 100644 index 000000000..c2589a219 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -0,0 +1,572 @@ +package org.labkey.sivstudies.etl; + +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationError; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.writer.ContainerUser; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class SubjectScopedSelect implements TaskRefTask +{ + protected final Map _settings = new CaseInsensitiveHashMap<>(); + protected ContainerUser _containerUser; + + private enum MODE + { + UPDATE_ONLY, + TRUNCATE; + } + + private enum Settings + { + subjectRemoteSource(false), + subjectSourceContainerPath(false), + subjectSourceSchema(true), + subjectSourceQuery(true), + subjectSourceColumn(true), + + dataRemoteSource(false), + dataSourceContainerPath(false), + dataSourceSchema(true), + dataSourceQuery(true), + dataSourceSubjectColumn(true), + dataSourceColumns(true), + dataSourceColumnMapping(false), + dataSourceAdditionalFilters(false), + dataSourceColumnDefaults(false), + + targetSchema(true), + targetQuery(true), + targetSubjectColumn(true), + targetAdditionalFilters(false); + + private final boolean _isRequired; + + Settings(boolean isRequired) + { + _isRequired = isRequired; + } + + public boolean isRequired() + { + return _isRequired; + } + } + + final int BATCH_SIZE = 100; + + private MODE getMode() + { + String rawVal = StringUtils.trimToNull(_settings.get("mode")); + if (rawVal == null) + { + return MODE.TRUNCATE; + } + + return MODE.valueOf(rawVal); + } + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + List subjects = getSubjects(job.getLogger()); + List> batches = Lists.partition(subjects, BATCH_SIZE); + job.getLogger().info("Total batches: " + batches.size()); + batches.forEach(x -> processBatch(x, job.getLogger())); + + return new RecordedActionSet(); + } + + private void processBatch(List subjects, Logger log) + { + log.info("processing batch with " + subjects.size() + " subjects"); + TableInfo destinationTable = getDataDestinationTable(); + + QueryUpdateService qus = destinationTable.getUpdateService(); + qus.setBulkLoad(true); + + try + { + if (getMode() == MODE.TRUNCATE) + { + // Find / Delete existing values: + Set keyFields = destinationTable.getColumns().stream().filter(ColumnInfo::isKeyField).collect(Collectors.toSet()); + final SimpleFilter subjectFilter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name())), subjects, CompareType.IN); + if (_settings.get(Settings.targetAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.targetAdditionalFilters.name())); + additionalFilters.forEach(subjectFilter::addCondition); + } + + if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null) + { + throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name())); + } + + Collection> existingRows = new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection(); + if (!existingRows.isEmpty()) + { + log.info("deleting " + existingRows.size() + " rows"); + qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), new ArrayList<>(existingRows), null, null); + } + else + { + log.info("No rows to delete for this subject batch"); + } + } + else + { + log.info("Using " + getMode().name() + " mode, source records will not be deleted"); + } + + // Query data and import + List> toImportOrUpdate = getRowsToImport(subjects, log); + if (!toImportOrUpdate.isEmpty()) + { + if (getMode() == MODE.TRUNCATE) + { + log.info("inserting " + toImportOrUpdate.size() + " rows"); + BatchValidationException bve = new BatchValidationException(); + qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImportOrUpdate, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + else if (getMode() == MODE.UPDATE_ONLY) + { + log.info("updating " + toImportOrUpdate.size() + " rows"); + BatchValidationException bve = new BatchValidationException(); + + Collection keyFields = destinationTable.getPkColumnNames(); + List> keys = toImportOrUpdate.stream().map(x -> { + Map map = new HashMap<>(); + for (String keyField : keyFields) + { + if (x.get(keyField) != null) + { + map.put(keyField, x.get(keyField)); + } + } + + return map; + }).toList(); + + qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), toImportOrUpdate, keys, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + else + { + throw new IllegalStateException("Unknown mode: " + getMode()); + } + } + else + { + log.info("No rows to import/update for this subject batch"); + } + } + catch (SQLException | InvalidKeyException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e) + { + throw new IllegalStateException("Error Importing/Updating Rows", e); + } + } + + private List parseAdditionalFilters(String rawVal) + { + rawVal = StringUtils.trimToNull(rawVal); + if (rawVal == null) + { + return Collections.emptyList(); + } + + SimpleFilter filter = new SimpleFilter(); + String[] filters = rawVal.split(";"); + for (String queryParam : filters) + { + filter.addUrlFilters(new ActionURL().setRawQuery(queryParam), null); + } + + return filter.getClauses().stream().map(fc -> { + if (fc instanceof CompareType.CompareClause cc) + { + return cc; + } + + throw new IllegalStateException("Expected all filters to be instance CompareType.CompareClause, found: " + fc.getClass()); + }).toList(); + } + + private Map parseSourceToDestColumnMap(String rawVal) + { + rawVal = StringUtils.trimToNull(rawVal); + if (rawVal == null) + { + return Collections.emptyMap(); + } + + Map colMap = new HashMap<>(); + + String[] tokens = rawVal.split(","); + for (String token : tokens) + { + if (!token.contains("=")) + { + throw new IllegalStateException("Invalid columnMapping: " + token); + } + + String[] els = token.split("="); + if (els.length != 2) + { + throw new IllegalStateException("Invalid columnMapping: " + token); + } + + colMap.put(els[0], els[1]); + } + + return colMap; + } + + private Map parseColumnDefaultMap(String rawVal) + { + rawVal = StringUtils.trimToNull(rawVal); + if (rawVal == null) + { + return Collections.emptyMap(); + } + + Map colMap = new HashMap<>(); + + String[] tokens = rawVal.split(";"); + for (String token : tokens) + { + if (!token.contains("=")) + { + throw new IllegalStateException("Invalid column defaultValue: " + token); + } + + String[] els = token.split("="); + if (els.length != 2) + { + throw new IllegalStateException("Invalid column defaultValue: " + token); + } + + colMap.put(els[0], els[1]); + } + + return colMap; + } + + private List> getRowsToImport(List subjects, Logger log) + { + if (_settings.get(Settings.dataSourceColumns.name()) == null) + { + throw new IllegalStateException("Missing value for dataSourceColumns"); + } + List sourceColumns = Arrays.asList(_settings.get(Settings.dataSourceColumns.name()).split(",")); + + Map sourceToDestColumMap = parseSourceToDestColumnMap(_settings.get(Settings.dataSourceColumnMapping.name())); + Map columnToDefaultMap = parseColumnDefaultMap(_settings.get(Settings.dataSourceColumnDefaults.name())); + + if (_settings.get(Settings.dataRemoteSource.name()) != null) + { + DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.dataRemoteSource.name()), log); + SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.dataSourceSchema.name()), _settings.get(Settings.dataSourceQuery.name())); + sr.setColumns(sourceColumns); + sr.addFilter(_settings.get(Settings.dataSourceSubjectColumn.name()), StringUtils.join(subjects, ";"), Filter.Operator.IN); + if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name())); + for (CompareType.CompareClause f : additionalFilters) + { + Object value; + if (f.getParamVals() == null) + { + value = null; + } + else if (f.getParamVals().length == 1) + { + value = f.getParamVals()[0]; + } + else + { + value = StringUtils.join(f.getParamVals(), ";"); + } + + Filter.Operator o = Filter.Operator.getOperatorFromUrlKey(f.getCompareType().getPreferredUrlKey()); + if (o == null) + { + throw new IllegalStateException("Unknown operator: " + f.getCompareType().getPreferredUrlKey() + ", raw filter: " + f.getCompareType().name()); + } + + sr.addFilter(new Filter(f.getFieldKey().toString(), value, o)); + } + } + + try + { + SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer); + + return doNameMapping(srr.getRows(), sourceToDestColumMap, columnToDefaultMap); + } + catch (CommandException | IOException e) + { + throw new IllegalStateException(e); + } + } + else + { + if (_settings.get(Settings.dataSourceContainerPath.name()) == null) + { + throw new IllegalStateException("Must provide dataSourceContainerPath for local sources"); + } + Container source = ContainerManager.getForPath(_settings.get(Settings.dataSourceContainerPath.name())); + if (source == null) + { + throw new IllegalStateException("Unknown container: " + _settings.get(Settings.dataSourceContainerPath.name())); + } + + if (!source.hasPermission(_containerUser.getUser(), ReadPermission.class)) + { + throw new IllegalStateException("User does not have read permission: " + _settings.get(Settings.dataSourceContainerPath.name())); + } + + UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), source, _settings.get(Settings.dataSourceSchema.name())); + if (us == null) + { + throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.dataSourceSchema.name())); + } + + TableInfo sourceTable = us.getTable(_settings.get(Settings.dataSourceQuery.name())); + if (sourceTable == null) + { + throw new IllegalStateException("Unknown table: " + _settings.get(Settings.dataSourceQuery.name())); + } + + if (!sourceTable.hasPermission(_containerUser.getUser(), ReadPermission.class)) + { + throw new IllegalStateException("User does not have read permission on the source table " + _settings.get(Settings.dataSourceContainerPath.name())); + } + + for (String colName : sourceColumns) + { + if (sourceTable.getColumn(colName) == null) + { + throw new IllegalStateException("Table is missing column: " + colName); + } + } + + if (sourceTable.getColumn(_settings.get(Settings.dataSourceSubjectColumn.name())) == null) + { + throw new IllegalStateException("Table is missing column: " + _settings.get(Settings.dataSourceSubjectColumn.name())); + } + + final SimpleFilter filter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.dataSourceSubjectColumn.name())), subjects, CompareType.IN); + if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name())); + additionalFilters.forEach(filter::addCondition); + } + + TableSelector ts = new TableSelector(sourceTable, new HashSet<>(sourceColumns), filter, null); + + return doNameMapping(new ArrayList<>(ts.getMapCollection()), sourceToDestColumMap, columnToDefaultMap); + } + } + + private List> doNameMapping(List> rows, Map colMap, Map columnToDefaultMap) + { + return rows.stream().map(row -> { + if (colMap.isEmpty()) + { + return row; + } + + colMap.forEach((sourceCol, destCol) -> { + if (row.containsKey(sourceCol)) + { + row.put(destCol, row.get(sourceCol)); + } + + row.remove(sourceCol); + }); + + return row; + }).map(row -> { + for (String colName : columnToDefaultMap.keySet()) + { + if (!row.containsKey(colName)) + { + row.put(colName, columnToDefaultMap.get(colName)); + } + else if (row.get(colName) == null) + { + row.put(colName, columnToDefaultMap.get(colName)); + } + } + + return row; + }).toList(); + } + + private TableInfo getDataDestinationTable() + { + UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), _settings.get(Settings.targetSchema.name())); + if (us == null) + { + throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.targetSchema.name())); + } + + TableInfo sourceTable = us.getTable(_settings.get(Settings.targetQuery.name())); + if (sourceTable == null) + { + throw new IllegalStateException("Unknown table: " + _settings.get(Settings.targetQuery.name())); + } + + return sourceTable; + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } + + @Override + public List preFlightCheck(Container c) + { + List errors = new ArrayList<>(); + for (String setting : getRequiredSettings()) + { + if (_settings.get(setting) == null || StringUtils.isEmpty(_settings.get(setting))) + { + errors.add(new SimpleValidationError("Missing required setting: " + setting)); + } + } + + return errors; + } + + @Override + public List getRequiredSettings() + { + return Arrays.stream(Settings.values()).filter(Settings::isRequired).map(Settings::name).toList(); + } + + @Override + public void setSettings(Map settings) + { + _settings.putAll(settings); + } + + private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Logger log) throws IllegalStateException + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, _containerUser.getContainer(), log); + if (rc == null) + { + throw new IllegalStateException("Unable to find remote connection: " + name); + } + + return rc; + } + + private List getSubjects(Logger log) + { + if (_settings.get(Settings.subjectRemoteSource.name()) != null) + { + DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.subjectRemoteSource.name()), log); + SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.subjectSourceSchema.name()), _settings.get(Settings.subjectSourceQuery.name())); + sr.setColumns(Arrays.asList(_settings.get(Settings.subjectSourceColumn.name()))); + + try + { + SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer); + + return srr.getRows().stream().map(x -> x.get(_settings.get(Settings.subjectSourceColumn.name()))).map(Object::toString).toList(); + } + catch (CommandException | IOException e) + { + throw new IllegalStateException(e); + } + } + else + { + Container source = _settings.get(Settings.subjectSourceContainerPath.name()) == null ? _containerUser.getContainer() : ContainerManager.getForPath(_settings.get(Settings.subjectSourceContainerPath.name())); + if (source == null) + { + throw new IllegalStateException("Unknown subjectSourceContainerPath: " + _settings.get(Settings.subjectSourceContainerPath.name())); + } + + if (!source.hasPermission(_containerUser.getUser(), ReadPermission.class)) + { + throw new IllegalStateException("User does not have read permission: " + _settings.get(Settings.subjectSourceContainerPath.name())); + } + + UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), source, _settings.get(Settings.subjectSourceSchema.name())); + if (us == null) + { + throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.subjectSourceSchema.name())); + } + + TableInfo sourceTable = us.getTable(_settings.get(Settings.subjectSourceQuery.name())); + if (sourceTable == null) + { + throw new IllegalStateException("Unknown table: " + _settings.get(Settings.subjectSourceQuery.name())); + } + + if (sourceTable.getColumn(_settings.get(Settings.subjectSourceColumn.name())) == null) + { + throw new IllegalStateException("Table is missing column: " + _settings.get(Settings.subjectSourceColumn.name())); + } + + return new TableSelector(sourceTable, PageFlowUtil.set(_settings.get(Settings.subjectSourceColumn.name()))).getArrayList(String.class); + } + } + + +} diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java new file mode 100644 index 000000000..b1b4de83e --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -0,0 +1,222 @@ +package org.labkey.sivstudies.query; + +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.WrappedColumn; +import org.labkey.api.ldk.table.AbstractTableCustomizer; +import org.labkey.api.query.LookupForeignKey; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryForeignKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.study.DatasetTable; +import org.labkey.api.util.logging.LogHelper; + +import java.util.ArrayList; +import java.util.List; + +public class SivStudiesCustomizer extends AbstractTableCustomizer +{ + public static final String ID_COL = "Id"; + private static final Logger _log = LogHelper.getLogger(SivStudiesCustomizer.class, "Table customization for the SIV Studies module"); + + @Override + public void customize(TableInfo tableInfo) + { + if (tableInfo instanceof DatasetTable ds) + { + performDatasetCustomization(ds); + } + } + + public void performDatasetCustomization(DatasetTable ds) + { + if (ds instanceof AbstractTableInfo ati) + { + _log.debug("Customizing dataset: {}", ds.getName()); + + if (!ds.getDataset().isDemographicData()) + { + appendAgeAtTimeCol(ds.getUserSchema(), ati, "date"); + } + + if ("demographics".equalsIgnoreCase(ds.getName())) + { + appendDemographicsColumns(ati); + } + } + else + { + _log.error("Expected DatasetTable to be instanceof AbstractTableInfo: " + ds.getName()); + } + } + + private ColumnInfo getPkCol(TableInfo ti) + { + List pks = ti.getPkColumns(); + return (pks.size() != 1) ? null : pks.get(0); + } + + private void appendAgeAtTimeCol(UserSchema demographicsSchema, AbstractTableInfo ds, final String dateColName) + { + String name = "ageAtTime"; + if (ds.getColumn(name, false) != null) + return; + + final ColumnInfo pkCol = getPkCol(ds); + if (pkCol == null) + return; + + final ColumnInfo idCol = ds.getColumn(ID_COL); + if (idCol == null) + return; + + if (ds.getColumn(dateColName) == null) + return; + + final String targetSchemaName = ds.getUserSchema().getName(); + final Container targetSchemaContainer = ds.getUserSchema().getContainer(); + final User u = ds.getUserSchema().getUser(); + final String schemaName = ds.getPublicSchemaName(); + final String queryName = ds.getName(); + final String demographicsPath = demographicsSchema.getContainer().getPath(); + + WrappedColumn col = new WrappedColumn(pkCol, name); + col.setLabel("Age At The Time"); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + col.setFk(new LookupForeignKey(){ + @Override + public TableInfo getLookupTableInfo() + { + String name = queryName + "_ageAtTime"; + UserSchema targetSchema = ds.getUserSchema().getDefaultSchema().getUserSchema(targetSchemaName); + QueryDefinition qd = QueryService.get().createQueryDef(u, targetSchemaContainer, targetSchema, name); + qd.setSql("SELECT\n" + + "c." + pkCol.getFieldKey().toSQLString() + ",\n" + + "\n" + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " ROUND(CONVERT(age_in_months(d.birth, d.death), DOUBLE) / 12, 1)\n" + + "ELSE\n" + + " ROUND(CONVERT(age_in_months(d.birth, CAST(c." + dateColName + " as DATE)), DOUBLE) / 12, 1)\n" + + "END AS float) as AgeAtTime,\n" + + "\n" + + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " ROUND(CONVERT(timestampdiff('SQL_TSI_DAY', d.birth, d.death), DOUBLE) / 365.25, 2)\n" + + "ELSE\n" + + " ROUND(CONVERT(timestampdiff('SQL_TSI_DAY', d.birth, CAST(c." + dateColName + " as DATE)), DOUBLE) / 365.25, 2)\n" + + "END AS float) as AgeAtTimeYears,\n" + + "\n" + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " floor(age(d.birth, d.death))\n" + + "ELSE\n" + + " floor(age(d.birth, CAST(c." + dateColName + " as DATE)))\n" + + "END AS float) as AgeAtTimeYearsRounded,\n" + + "\n" + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY',d.birth, d.death), INTEGER)\n" + + "ELSE\n" + + " CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY',d.birth, CAST(c." + dateColName + " AS DATE)), INTEGER)\n" + + "END AS float) as AgeAtTimeDays,\n" + + "\n" + + "CAST(\n" + + "CASE\n" + + "WHEN d.birth is null or c." + dateColName + " is null\n" + + " THEN null\n" + + "WHEN (d.death IS NOT NULL AND d.death < c." + dateColName + ") THEN\n" + + " CONVERT(age_in_months(d.birth, d.death), INTEGER)\n" + + "ELSE\n" + + " CONVERT(age_in_months(d.birth, CAST(c." + dateColName + " AS DATE)), INTEGER)\n" + + "END AS float) as AgeAtTimeMonths,\n" + + "FROM \"" + schemaName + "\".\"" + queryName + "\" c " + + "LEFT JOIN \"" + demographicsPath + "\".study.demographics d ON (d.Id = c." + idCol.getFieldKey().toSQLString() + ")" + ); + qd.setIsTemporary(true); + + List errors = new ArrayList<>(); + TableInfo ti = qd.getTable(errors, true); + if (!errors.isEmpty()) + { + _log.warn("Error creating age at time lookup table for: " + schemaName + "." + queryName + " in container: " + targetSchema.getContainer().getPath()); + for (QueryException e : errors) + { + _log.warn(e.getMessage(), e); + } + } + + if (ti != null) + { + ((BaseColumnInfo)ti.getColumn(pkCol.getName())).setHidden(true); + ((BaseColumnInfo)ti.getColumn(pkCol.getName())).setKeyField(true); + } + + return ti; + } + }); + + ds.addColumn(col); + } + + private void appendDemographicsColumns(AbstractTableInfo demographicsTable) + { + if (demographicsTable.getColumn("mhcGenotypes") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(demographicsTable.getUserSchema(), "demographicsMHC", demographicsTable, "mhcGenotypes"); + colInfo.setLabel("MHC Genotypes"); + demographicsTable.addColumn(colInfo); + } + } + + private void appendPvlColumns(AbstractTableInfo ti, ColumnInfo subjectCol, ColumnInfo dateCol) + { + Container target = ti.getUserSchema().getContainer().isWorkbookOrTab() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer(); + +// TableInfo data = schema.createDataTable(null); +// String tableName = data.getDomain().getStorageTableName(); +// +// String name = "viralLoad"; +// if (ti.getColumn(name) == null) +// { +// SQLFragment sql = new SQLFragment("(SELECT avg(viralLoad) as expr FROM assayresult." + tableName + " t WHERE t.subjectId = " + ExprColumn.STR_TABLE_ALIAS + "." + subjectCol.getName() + " AND CAST(t.date AS DATE) = CAST(" + ExprColumn.STR_TABLE_ALIAS + "." + dateCol.getName() + " AS DATE))"); +// ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.DOUBLE, subjectCol, dateCol); +// newCol.setDescription("Displays the viral load from this timepoint, if present"); +// newCol.setLabel("Viral Load (copies/mL)"); +// ti.addColumn(newCol); +// } + } + + private BaseColumnInfo getWrappedIdCol(UserSchema targetQueryUserSchema, String targetQueryName, AbstractTableInfo demographicsTable, String colName) + { + WrappedColumn col = new WrappedColumn(demographicsTable.getColumn(ID_COL), colName); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + col.setFk(new QueryForeignKey(demographicsTable.getUserSchema(), null, targetQueryUserSchema, null, targetQueryName, ID_COL, ID_COL)); + + return col; + } +} diff --git a/SivStudies/test/src/org/labkey/test/tests/sivstudies/SivStudiesTest.java b/SivStudies/test/src/org/labkey/test/tests/sivstudies/SivStudiesTest.java new file mode 100644 index 000000000..59575f2c9 --- /dev/null +++ b/SivStudies/test/src/org/labkey/test/tests/sivstudies/SivStudiesTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2025 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.tests.sivstudies; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.TestTimeoutException; +import org.labkey.test.categories.InDevelopment; + +import java.util.Collections; +import java.util.List; + +/** + * ~ Created from 'moduleTemplate' ~ + * This test validates basic functionality of the SivStudies module + */ +@Category({InDevelopment.class}) +public class SivStudiesTest extends BaseWebDriverTest +{ + @Override + protected void doCleanup(boolean afterTest) throws TestTimeoutException + { + _containerHelper.deleteProject(getProjectName(), afterTest); + } + + @BeforeClass + public static void setupProject() + { + SivStudiesTest init = getCurrentTest(); + + init.doSetup(); + } + + private void doSetup() + { + _containerHelper.createProject(getProjectName(), null); + } + + @Before + public void preTest() + { + goToProjectHome(); + } + + @Test + public void testSivStudiesModule() + { + _containerHelper.enableModule("SivStudies"); + + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } + + @Override + protected String getProjectName() + { + return "SivStudiesTest Project"; + } + + @Override + public List getAssociatedModules() + { + return Collections.singletonList("SivStudies"); + } +} \ No newline at end of file diff --git a/mGAP/resources/views/mgapDataDashboard.html b/mGAP/resources/views/mgapDataDashboard.html index 71a537807..87cf98a9c 100644 --- a/mGAP/resources/views/mgapDataDashboard.html +++ b/mGAP/resources/views/mgapDataDashboard.html @@ -168,6 +168,14 @@ schemaName: 'mgap', queryName: 'duplicategvcfs' }) + },{ + name: 'CRAMs Not Using Archival Mode', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, { + schemaName: 'sequenceanalysis', + queryName: 'outputfiles', + 'query.dataid/DataFileUrl~contains': '.cram', + 'query.description~doesnotcontain': 'Archival' + }) }] }] }] diff --git a/mGAP/resources/views/overview.html b/mGAP/resources/views/overview.html index 65ada0fff..9da48179d 100644 --- a/mGAP/resources/views/overview.html +++ b/mGAP/resources/views/overview.html @@ -46,7 +46,7 @@

Structural Variant Catalog: We have released a draft dataset with structural variants generated from PacBio sequencing of 44 Rhesus macaques. These data complement the short variant catalog by detecting categories of variants not readily accomplished with short read Illumina data.

NHP Models of Human Disease: Phenotypes

diff --git a/mGAP/resources/web/mGAP/DownloadWindow.js b/mGAP/resources/web/mGAP/DownloadWindow.js index 69ff0a824..f18dffef7 100644 --- a/mGAP/resources/web/mGAP/DownloadWindow.js +++ b/mGAP/resources/web/mGAP/DownloadWindow.js @@ -58,21 +58,26 @@ Ext4.define('mGAP.window.DownloadWindow', { var urlFasta = results.rows[0]['vcfId/library_id/fasta_file/Name']; var sitesOnlyVcf = results.rows[0].objectId + '/' + results.rows[0]['sitesOnlyVcfId/dataid/Name']; + const getFileName = function(path){ + path = path.split('/'); + return(path[path.length - 1]); + } + var toAdd = [{ html: 'Due to the large file size, the preferred option is to download using wget or curl on the command line, such as the exmaples below. Nonetheless, you also are able to paste the URLs into your browser and download through this way as well, although it will be slower and possibly not able to resume if your connection is disrupted.

' + - 'Use these to download the VCF and index:
' + - '
wget https://mgapdownload.ohsu.edu/' + releaseVcf + '
' +
-                            'wget https://mgapdownload.ohsu.edu/' + releaseVcf + '.tbi
' + + 'Use these to download the VCF and index. Note, -C will allow file resume if the download is disrupted:
' + + '
curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/' + releaseVcf + '
' +
+                            'curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/' + releaseVcf + '.tbi
' + (sitesOnlyVcf ? 'or a VCF without genotypes (considerably smaller):
' + - '
wget https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '
' +
-                            'wget https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '.tbi
' : '') + + '
curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '
' +
+                            'curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/' + sitesOnlyVcf + '.tbi
' : '') + 'and genome:
' + - '
wget https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '
' +
-                            'wget https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '.fai
' +
-                            'wget https://mgapdownload.ohsu.edu/genomes/' + urlFasta.replace(/fasta$/, 'dict') + '
' + '
curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '
' +
+                            'curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/genomes/' + urlFasta + '.fai
' +
+                            'curl -C - -o ' + getFileName(releaseVcf) + ' https://mgapdownload.ohsu.edu/genomes/' + urlFasta.replace(/fasta$/, 'dict') + '
' },{ - html: '
mGAP is an NIH funded project. If you use these data in a publication, we ask that you please include R24OD021324 in the acknowledgements.', + html: '
mGAP is an NIH funded project. If you use these data in a publication, we ask that you please include R24 OD021324 in the acknowledgements.', border: false, style: 'padding-bottom: 20px;' }]; diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java index 04530b420..184f21042 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java @@ -133,7 +133,7 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S File localCopy = doFileCopy(f, subDir, name); if (localCopy == null) { - // TODO + getStatusLogger().warn("localCopy was null", new Exception()); } //first create the ExpData @@ -217,7 +217,6 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip File localCopy = new File(subdir, name == null || f.getName().startsWith("mGap.v") ? f.getName() : FileUtil.makeLegalName(name).replaceAll(" ", "_") + ".vcf.gz"); if (f.equals(localCopy)) { - getStatusLogger().debug("Attempting to copy file that is already a child of the target: " + f.getPath(), new Exception()); return localCopy; } diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java index 4199f61fa..b90b4d201 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java @@ -54,17 +54,18 @@ protected String getGenomeIdField() } @Override - protected void addTracks(final String databaseId, String releaseId) + protected boolean addTracks(final String databaseId, String releaseId) { try { getStatusLogger().info("possibly creating track for: " + getDatabaseName()); String jsonFile = getOrCreateJsonFile(); - getOrCreateDatabaseMember(databaseId, jsonFile); + return getOrCreateDatabaseMember(databaseId, jsonFile); } catch(Exception e) { getStatusLogger().error(e.getMessage(), e); + return false; } } diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java index e489595de..04e11f67a 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java @@ -4,13 +4,13 @@ import org.json.JSONObject; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; import org.labkey.api.data.Results; -import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Sort; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.jbrowse.JBrowseService; @@ -30,8 +30,11 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; /** * Created by bimber on 5/15/2017. @@ -66,44 +69,48 @@ protected Object doTransform(Object inputValue) Integer outputFileId = getOrCreateOutputFile(input, getInputValue("objectId"), null); if (outputFileId != null) { - //find database ID, if exists: - //determine if there is already a JSONfile for this outputfile + //find database ID, if exists, based on name: UserSchema us = getJbrowseUserSchema(); - SQLFragment sql = new SQLFragment("SELECT m." + us.getDbSchema().getScope().getSqlDialect().makeLegalIdentifier("database") + " FROM jbrowse.jsonfiles j JOIN jbrowse.database_members m ON (j.objectId = m.jsonfile) WHERE j.outputfile = ?", outputFileId); - String databaseId = new SqlSelector(us.getDbSchema().getScope(), sql).getObject(String.class); - if (databaseId != null) + TableSelector ts = new TableSelector(us.getTable("databases"), PageFlowUtil.set("objectid"), new SimpleFilter(FieldKey.fromString("name"), getDatabaseName()), new Sort("-rowid")); + if (ts.exists()) { - getStatusLogger().info("jbrowse database exists using the output file: " + outputFileId); + String databaseId = ts.getArrayList(String.class).stream().toList().get(0); + getStatusLogger().info("jbrowse database exists using name: " + getDatabaseName()); + + boolean hadChanges = addTracks(databaseId, releaseId); + if (hadChanges) + { + recreateSession(databaseId); + } + return databaseId; } - else + + String databaseId = new GUID().toString(); + try { - try - { - databaseId = new GUID().toString(); - getStatusLogger().info("creating jbrowse database: " + databaseId); - - //create database - TableInfo databases = getJbrowseUserSchema().getTable("databases"); - CaseInsensitiveHashMap dbRow = new CaseInsensitiveHashMap<>(); - dbRow.put("objectid", databaseId); - dbRow.put("name", getDatabaseName()); - dbRow.put("description", null); - dbRow.put("libraryId", getLibraryId()); - dbRow.put("temporary", false); - dbRow.put("container", getContainerUser().getContainer().getId()); - dbRow.put("created", new Date()); - dbRow.put("createdby", getContainerUser().getUser().getUserId()); - dbRow.put("modified", new Date()); - dbRow.put("modifiedby", getContainerUser().getUser().getUserId()); - dbRow.put("jsonConfig", getSessionJson()); - - databases.getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(dbRow), new BatchValidationException(), null, new HashMap<>()); - } - catch (Exception e) - { - getStatusLogger().error("Error creating database: " + inputValue, e); - } + getStatusLogger().info("creating jbrowse database: " + databaseId + ", for output file: " + outputFileId); + + //create database + TableInfo databases = getJbrowseUserSchema().getTable("databases"); + CaseInsensitiveHashMap dbRow = new CaseInsensitiveHashMap<>(); + dbRow.put("objectid", databaseId); + dbRow.put("name", getDatabaseName()); + dbRow.put("description", null); + dbRow.put("libraryId", getLibraryId()); + dbRow.put("temporary", false); + dbRow.put("container", getContainerUser().getContainer().getId()); + dbRow.put("created", new Date()); + dbRow.put("createdby", getContainerUser().getUser().getUserId()); + dbRow.put("modified", new Date()); + dbRow.put("modifiedby", getContainerUser().getUser().getUserId()); + dbRow.put("jsonConfig", getSessionJson()); + + databases.getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(dbRow), new BatchValidationException(), null, new HashMap<>()); + } + catch (Exception e) + { + getStatusLogger().error("Error creating database: " + inputValue, e); } addTracks(databaseId, releaseId); @@ -140,7 +147,7 @@ private void recreateSession(final String databaseId) }, DbScope.CommitTaskOption.POSTCOMMIT); } - protected void addTracks(final String databaseId, String releaseId) + protected boolean addTracks(final String databaseId, String releaseId) { //then JSONfiles/database members List fks = Arrays.asList( @@ -166,21 +173,51 @@ protected void addTracks(final String databaseId, String releaseId) getStatusLogger().error("no track records found for release: " + releaseId); } + final AtomicBoolean hadChanges = new AtomicBoolean(false); + final Set jsonFiles = new HashSet<>(); ts.forEachResults(rs -> { try { getStatusLogger().info("possibly creating track for: " + rs.getString(FieldKey.fromString("trackName"))); String jsonFile = getOrCreateJsonFile(rs, "vcfId/dataid/DataFileUrl"); - getOrCreateDatabaseMember(databaseId, jsonFile); + jsonFiles.add(jsonFile); + boolean added = getOrCreateDatabaseMember(databaseId, jsonFile); + if (added) + { + hadChanges.set(true); + } } catch (Exception e) { getStatusLogger().error(e.getMessage(), e); } }); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("database"), databaseId); + filter.addCondition(FieldKey.fromString("jsonfile"), jsonFiles, CompareType.NOT_IN); + TableSelector ts2 = new TableSelector(getDatabaseMembers(), PageFlowUtil.set("rowid"), filter, null); + if (ts2.exists()) + { + List rowIds = ts2.getArrayList(Integer.class); + + getStatusLogger().info("Deleting " + rowIds.size() + " database_member rows for: " + databaseId); + List> toDelete = new ArrayList<>(); + rowIds.forEach(rowId -> toDelete.add(new CaseInsensitiveHashMap<>(Map.of("rowid", rowId)))); + try + { + getJbrowseUserSchema().getTable("database_members").getUpdateService().deleteRows(getContainerUser().getUser(), getContainerUser().getContainer(), toDelete, null, null); + hadChanges.set(true); + } + catch (InvalidKeyException | BatchValidationException | SQLException |QueryUpdateServiceException e) + { + getStatusLogger().error(e); + } + } + + return hadChanges.get(); } - private void ensureLuceneData(String objectId) + private void ensureLuceneData(String objectId, boolean hasIndex) { //determine if there is already a JSONfile for this outputfile TableSelector ts1 = new TableSelector(getJsonFiles(), PageFlowUtil.set("container"), new SimpleFilter(FieldKey.fromString("objectid"), objectId), null); @@ -197,7 +234,7 @@ private void ensureLuceneData(String objectId) Map row = new CaseInsensitiveHashMap<>(); row.put("objectid", objectId); row.put("container", containerId); - row.put("trackJson", getTrackJson(true)); + row.put("trackJson", getTrackJson(hasIndex)); TableInfo jsonFiles = getJbrowseUserSchema().getTable("jsonfiles"); jsonFiles.getUpdateService().updateRows(getContainerUser().getUser(), getContainerUser().getContainer(), Arrays.asList(row), Arrays.asList(new CaseInsensitiveHashMap<>(Map.of("objectid", objectId))), new BatchValidationException(), null, null); @@ -208,7 +245,7 @@ private void ensureLuceneData(String objectId) } } - protected void getOrCreateDatabaseMember(String databaseId, String jsonFileId) throws Exception + protected boolean getOrCreateDatabaseMember(String databaseId, String jsonFileId) throws Exception { SimpleFilter filter = new SimpleFilter(FieldKey.fromString("database"), databaseId); filter.addCondition(FieldKey.fromString("jsonfile"), jsonFileId); @@ -216,7 +253,7 @@ protected void getOrCreateDatabaseMember(String databaseId, String jsonFileId) t if (new TableSelector(getDatabaseMembers(), filter, null).exists()) { getStatusLogger().info("database member exists for: " + jsonFileId); - return; + return false; } TableInfo databaseMembers = getJbrowseUserSchema().getTable("database_members"); @@ -232,6 +269,8 @@ protected void getOrCreateDatabaseMember(String databaseId, String jsonFileId) t getStatusLogger().info("creating database member for: " + jsonFileId); databaseMembers.getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(row), new BatchValidationException(), null, new HashMap<>()); + + return true; } protected TableInfo getJsonFiles() @@ -296,7 +335,8 @@ private String getOrCreateJsonFile(Results rs, String fieldKey) throws SQLExcept String objectId = ts1.getArrayList(String.class).get(0); if (isDefaultTrack) { - ensureLuceneData(objectId); + boolean expectIndex = rs.getObject(FieldKey.fromString("releaseId/luceneIndex")) != null || rs.getObject(FieldKey.fromString("vcfIndexId")) != null; + ensureLuceneData(objectId, expectIndex); } return objectId; diff --git a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java index 373e4a30e..94aa33734 100644 --- a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java +++ b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java @@ -1,5 +1,6 @@ package org.labkey.mgap; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; @@ -25,12 +26,16 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; public class mGapMaintenanceTask implements SystemMaintenance.MaintenanceTask @@ -89,6 +94,34 @@ private void checkMGapFiles(Logger log, User u) return; } + Map> versionMap = new HashMap<>(); + new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("species", "version", "objectid")).forEachResults(rs -> { + String species = rs.getString(FieldKey.fromString("species")); + if (!versionMap.containsKey(species)) + { + versionMap.put(species, new TreeMap<>()); + } + + String objectId = rs.getString(FieldKey.fromString("objectid")); + String versionString = rs.getString(FieldKey.fromString("version")); + if (NumberUtils.isCreatable(versionString)) + { + Double version = Double.parseDouble(versionString); + versionMap.get(species).put(version, objectId); + } + else + { + log.error("Non-numeric version: " + versionString); + } + }); + + Map latestReleaseBySpecies = new HashMap<>(); + for (String species : versionMap.keySet()) + { + Double maxVersion = versionMap.get(species).keySet().stream().sorted().max(Double::compareTo).get(); + latestReleaseBySpecies.put(species, versionMap.get(species).get(maxVersion)); + } + // Find expected folder names: List releaseIds = new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("objectid")).getArrayList(String.class); @@ -112,7 +145,7 @@ private void checkMGapFiles(Logger log, User u) } List commandsToRun = new ArrayList<>(); - releaseIds.forEach(f -> inspectReleaseFolder(f, baseDir, c, u, log, toDelete, commandsToRun)); + releaseIds.forEach(f -> inspectReleaseFolder(f, baseDir, c, u, log, toDelete, commandsToRun, latestReleaseBySpecies)); // Also verify genomes: Set genomesIds = new HashSet<>(new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("genomeId")).getArrayList(Integer.class)); @@ -150,7 +183,7 @@ private void checkMGapFiles(Logger log, User u) } } - private void inspectReleaseFolder(String releaseId, File baseDir, Container c, User u, final Logger log, final Set toDelete, List commandsToRun) + private void inspectReleaseFolder(String releaseId, File baseDir, Container c, User u, final Logger log, final Set toDelete, List commandsToRun, Map speciesToLatestTrack) { File releaseDir = new File(baseDir, releaseId); if (!releaseDir.exists()) @@ -159,6 +192,9 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U return; } + String species = new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("species"), new SimpleFilter(FieldKey.fromString("objectid"), releaseId), null).getObject(String.class); + boolean isLatestReleaseForSpecies = releaseId.equals(speciesToLatestTrack.get(species)); + final Set expectedFiles = new HashSet<>(); List tracksFromRelease = new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_TRACKS_PER_RELEASE), PageFlowUtil.set("vcfId"), new SimpleFilter(FieldKey.fromString("releaseId"), releaseId), null).getArrayList(Integer.class); tracksFromRelease.forEach(rowId -> { @@ -180,6 +216,11 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U checkSymlink(log, f, releaseId, commandsToRun); expectedFiles.add(new File(f.getPath() + ".tbi")); checkSymlink(log, new File(f.getPath() + ".tbi"), releaseId, commandsToRun); + + if (isLatestReleaseForSpecies) + { + checkSymlink(log, f, releaseId, commandsToRun); + } }); final Set fields = PageFlowUtil.set("vcfId", "variantTable", "liftedVcfId", "sitesOnlyVcfId", "novelSitesVcfId", "luceneIndex"); @@ -218,10 +259,18 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U expectedFiles.add(new File(f.getPath() + ".tbi")); } - if ("sitesOnlyVcfId".equals(field)) + if ("vcfId".equals(field) || "sitesOnlyVcfId".equals(field)) { checkSymlink(log, f, releaseId, commandsToRun); checkSymlink(log, new File(f.getPath() + ".tbi"), releaseId, commandsToRun); + + if (isLatestReleaseForSpecies & "sitesOnlyVcfId".equals(field)) + { + File symlinkTarget = new File("/var/www/html/latest/" + "mGAP." + species.replaceAll(" ", "_") + ".vcf.gz"); + checkSymlink(log, f, symlinkTarget, commandsToRun); + + checkSymlink(log, new File(f.getPath() + ".tbi"), new File(symlinkTarget.getPath() + ".tbi"), commandsToRun); + } } } }); @@ -262,13 +311,63 @@ private void checkSymlink(Logger log, File f, @Nullable String dirName, List commandsToRun) + { + if (!expectedSymlink.getParentFile().exists()) + { + commandsToRun.add("mkdir -p " + expectedSymlink.getParentFile()); + } + if (!expectedSymlink.exists()) { log.error("Missing symlink: " + expectedSymlink.getPath()); - commandsToRun.add("ln -s " + f.getPath() + " " + expectedSymlink.getPath()); + log.error("to path: " + f.getPath()); + + File target = resolveSymlink(f); + commandsToRun.add("ln -s " + target.getPath() + " " + expectedSymlink.getPath()); + } + else + { + if (!Files.isSymbolicLink(expectedSymlink.toPath())) + { + log.error("File is not a symlink: "+ expectedSymlink.getPath()); + } + else + { + File expectedTarget = resolveSymlink(f); + File actualTarget = resolveSymlink(expectedSymlink); + if (!expectedTarget.equals(actualTarget)) + { + log.error("Symlink does not point to the correct location: " + expectedSymlink.getPath()); + commandsToRun.add("rm " + expectedSymlink.getPath()); + + commandsToRun.add("ln -s " + expectedTarget.getPath() + " " + expectedSymlink.getPath()); + } + } } } + private File resolveSymlink(File sourceFile) + { + File target = sourceFile; + if (Files.isSymbolicLink(target.toPath())) + { + try + { + target = Files.readSymbolicLink(target.toPath()).toFile(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + return target; + } + private void checkForDuplicateAliases(Logger log, User u) { // Find all container Ids in use in the table: @@ -279,6 +378,7 @@ private void checkForDuplicateAliases(Logger log, User u) if (c == null) { log.error("Unable to find container: " + containerId); + return; } if (!c.getActiveModules().contains(ModuleLoader.getInstance().getModule(mGAPModule.class))) diff --git a/mcc/package-lock.json b/mcc/package-lock.json index 7ad10c48b..542c41cf7 100644 --- a/mcc/package-lock.json +++ b/mcc/package-lock.json @@ -8157,9 +8157,9 @@ } }, "node_modules/tar-fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", - "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", + "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", "dev": true, "dependencies": { "chownr": "^1.1.1", diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index f93799522..9e3d71ad9 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -95,23 +95,6 @@ - - Copy to target - - - Id - date - category - observation - objectid - - - - - - - - Copy to target @@ -153,6 +136,23 @@ + + Copy to target + + + Id + date + category + observation + objectid + + + + + + + + diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index af4095cf0..7bf969e39 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -23,6 +23,7 @@ import org.labkey.api.data.Container; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.ldk.LDKService; +import org.labkey.api.ldk.notification.NotificationService; import org.labkey.api.module.ModuleContext; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; @@ -32,6 +33,7 @@ import org.labkey.api.view.WebPartFactory; import org.labkey.primeseq.analysis.CombineMethylationRatesHandler; import org.labkey.primeseq.analysis.MethylationRateComparisonHandler; +import org.labkey.primeseq.notification.DiskUsageNotification; import org.labkey.primeseq.pipeline.BismarkWrapper; import org.labkey.primeseq.pipeline.BlastPipelineJobResourceAllocator; import org.labkey.primeseq.pipeline.ClusterMaintenanceTask; @@ -85,6 +87,8 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) LDKService.get().registerQueryButton(new UpdateResourcesButton(), "pipeline", "job"); LDKService.get().registerQueryButton(new PerformMhcCleanupButton(), "sequenceanalysis", "sequence_analyses"); LDKService.get().registerQueryButton(new DeleteJobCheckpointButton(), "pipeline", "job"); + + NotificationService.get().registerNotification(new DiskUsageNotification()); } @Override diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java new file mode 100644 index 000000000..7245af41c --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -0,0 +1,199 @@ +package org.labkey.primeseq.notification; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.SystemUtils; +import org.apache.commons.lang3.time.DurationFormatUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.ldk.notification.Notification; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.api.settings.LookAndFeelProperties; + +import java.text.DateFormat; +import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * User: bbimber + * Date: 8/4/12 + */ +public class DiskUsageNotification implements Notification +{ + protected final static Logger _log = LogManager.getLogger(DiskUsageNotification.class); + private static final String lastSave = "lastSave"; + private NumberFormat _pctFormat = null; + + private static final String PROP_CATEGORY = "primeseq.DiskUsageNotification"; + + @Override + public String getName() + { + return "Disk/Cluster Usage Notification"; + } + + @Override + public String getCategory() + { + return "Admin"; + } + + @Override + public boolean isAvailable(Container c) + { + return true; + } + + @Override + public String getDescription() + { + return "This runs once a week to summarize file system and cluster usage"; + } + + @Override + public String getEmailSubject(Container c) + { + return "Disk/Cluster Usage: " + getDateTimeFormat(c).format(new Date()); + } + + public DateFormat getDateTimeFormat(Container c) + { + return new SimpleDateFormat(LookAndFeelProperties.getInstance(c).getDefaultDateTimeFormat()); + } + + @Override + public String getCronString() + { + return "0 8 * * 1 ?"; + } + + @Override + public String getScheduleDescription() + { + return "Every Monday at 8AM"; + } + + private Map getSavedValues(Container c) + { + return PropertyManager.getProperties(c, PROP_CATEGORY); + } + + @Override + public String getMessageBodyHTML(Container c, User u) + { + Date start = new Date(); + + _pctFormat = NumberFormat.getPercentInstance(); + _pctFormat.setMaximumFractionDigits(1); + + StringBuilder msg = new StringBuilder(); + getDiskUsageStats(c, u, msg); + getClusterUsage(c, u, msg); + + msg.insert(0, "This email summarizes disk and cluster usage. It was run on: " + getDateTimeFormat(c).format(new Date()) + ". Runtime: " + DurationFormatUtils.formatDurationWords((new Date()).getTime() - start.getTime(), true, true) + "

"); + + return msg.toString(); + } + + private void getClusterUsage(Container c, User u, final StringBuilder msg) + { + if (!SystemUtils.IS_OS_LINUX) + { + return; + } + + try + { + SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); + String results = wrapper.executeWithOutput(Arrays.asList("ssh", "-q", "labkey_submit@arc", "sshare", "-U", "-u", "labkey_submit")); + + msg.append("Cluster Usage:

"); + msg.append(""); + + AtomicBoolean foundHeader = new AtomicBoolean(false); + Arrays.stream(results.split("\n")).forEach(x -> { + if (x.startsWith("------------")) + { + foundHeader.set(true); + return; + } + else if (!foundHeader.get()) + { + return; + } + + String[] els = x.split("[ ]+"); + + if (els.length != 7) + { + _log.error("Unexpected line: " + StringUtils.join(els, "<>")); + return; + } + + msg.append(""); + msg.append(""); + msg.append(""); + long shares = StringUtils.isEmpty(els[4]) ? 0 : Long.parseLong(els[4]); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + }); + msg.append("
AccountNormSharesRawUsageEffectiveUsageFairShare
").append(els[0]).append("").append(els[3]).append("").append(String.format("%,d", shares)).append("").append(els[5]).append("").append(els[6]).append("
"); + } + catch (PipelineJobException e) + { + _log.error("Error fetching slurm summary", e); + } + + msg.append("

\n"); + + } + + private void getDiskUsageStats(Container c, User u, final StringBuilder msg) + { + if (!SystemUtils.IS_OS_LINUX) + { + return; + } + + try + { + SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); + String results = wrapper.executeWithOutput(Arrays.asList("df", "-h", "/home/groups/BimberLab/", "/home/groups/OnprcColonyData/", "/home/groups/prime-seq/", "/home/exacloud/gscratch/prime-seq/")); + + msg.append("Disk Usage Stats:

"); + msg.append(""); + Arrays.stream(results.split("\n")).forEach(x -> { + String[] els = x.split("[ ]+"); + if ("Filesystem".equalsIgnoreCase(els[0])) + { + return; + } + + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + msg.append(""); + }); + msg.append("
FilesystemSizeUsedAvailablePercent
").append(els[0]).append("").append(els[1]).append("").append(els[2]).append("").append(els[3]).append("").append(els[4]).append("
"); + } + catch (PipelineJobException e) + { + _log.error("Error running df", e); + } + + msg.append("

\n"); + } +} \ No newline at end of file diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index 34b7bbb76..1151b766f 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -82,6 +82,24 @@ private boolean isSequenceSequenceOutputHandlerTask(PipelineJob job) private Long _totalFileSize = null; private static final Long UNABLE_TO_DETERMINE = -1L; + private int getAlignerIndexMem(PipelineJob job) + { + if (job instanceof HasJobParams) + { + Map params = ((HasJobParams)job).getJobParams(); + if (params.get("alignment") != null) + { + String aligner = params.get("alignment"); + if (Arrays.asList("BWA-Mem", "BWA-Mem2", "STAR").contains(aligner)) + { + return 72; + } + } + } + + return 36; + } + @Override public Integer getMaxRequestCpus(PipelineJob job) { @@ -165,11 +183,11 @@ public Integer getMaxRequestMemory(PipelineJob job) return 72; } - // NOTE: STAR in particular needs more memory if (isCacheAlignerIndexesTask(job)) { - job.getLogger().debug("setting memory to 36"); - return 36; + int mem = getAlignerIndexMem(job); + job.getLogger().debug("setting memory to: " + mem); + return mem; } if (isLuceneIndexJob(job)) diff --git a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml index f64b4c073..2ac1c14cd 100644 --- a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml +++ b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml @@ -18,7 +18,6 @@ - @@ -27,6 +26,5 @@ - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/stims/.qview.xml b/tcrdb/resources/queries/tcrdb/stims/.qview.xml new file mode 100644 index 000000000..65170b8d8 --- /dev/null +++ b/tcrdb/resources/queries/tcrdb/stims/.qview.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.52-15.53.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.52-15.53.sql new file mode 100644 index 000000000..cd444bfe1 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.52-15.53.sql @@ -0,0 +1,23 @@ +drop table tcrdb.stims; +drop table tcrdb.sorts; +drop table tcrdb.cdnas; +drop table tcrdb.peptides; +drop table tcrdb.plate_processing; +drop table tcrdb.citeseq_panels; +drop table tcrdb.citeseq_antibodies; + +ALTER TABLE tcrdb.clone_responses DROP COLUMN experiment; +ALTER TABLE tcrdb.clone_responses DROP COLUMN cloneName; +ALTER TABLE tcrdb.clone_responses DROP COLUMN date; +ALTER TABLE tcrdb.clone_responses DROP COLUMN cellBackground; +ALTER TABLE tcrdb.clone_responses DROP COLUMN numEffectors; +ALTER TABLE tcrdb.clone_responses DROP COLUMN pctTransduction; +ALTER TABLE tcrdb.clone_responses DROP COLUMN costim; +ALTER TABLE tcrdb.clone_responses DROP COLUMN antigen; +ALTER TABLE tcrdb.clone_responses DROP COLUMN stim; + +ALTER TABLE tcrdb.clone_responses ADD COLUMN cdna_id int; +ALTER TABLE tcrdb.clone_responses ADD COLUMN nostimid int; +ALTER TABLE tcrdb.clone_responses ADD COLUMN chain varchar(100); +ALTER TABLE tcrdb.clone_responses ADD COLUMN clonotype varchar(1000); + diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.53-15.54.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.53-15.54.sql new file mode 100644 index 000000000..743dffcbd --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.53-15.54.sql @@ -0,0 +1,20 @@ +CREATE TABLE tcrdb.stims ( + rowid SERIAL, + cdna_id int, + controlStimId int, + + quantificationMethod varchar(1000), + quantification double precision, + + flowQuantificationMethod varchar(1000), + flowQuantification double precision, + comment varchar(4000), + + container entityid, + created timestamp, + createdby int, + modified timestamp, + modifiedby int, + + constraint PK_stims PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql new file mode 100644 index 000000000..2f92b465b --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.52-15.53.sql @@ -0,0 +1,23 @@ +drop table tcrdb.stims; +drop table tcrdb.sorts; +drop table tcrdb.cdnas; +drop table tcrdb.peptides; +drop table tcrdb.plate_processing; +drop table tcrdb.citeseq_panels; +drop table tcrdb.citeseq_antibodies; + +ALTER TABLE tcrdb.clone_responses DROP COLUMN experiment; +ALTER TABLE tcrdb.clone_responses DROP COLUMN cloneName; +ALTER TABLE tcrdb.clone_responses DROP COLUMN date; +ALTER TABLE tcrdb.clone_responses DROP COLUMN cellBackground; +ALTER TABLE tcrdb.clone_responses DROP COLUMN numEffectors; +ALTER TABLE tcrdb.clone_responses DROP COLUMN pctTransduction; +ALTER TABLE tcrdb.clone_responses DROP COLUMN costim; +ALTER TABLE tcrdb.clone_responses DROP COLUMN antigen; +ALTER TABLE tcrdb.clone_responses DROP COLUMN stim; + +ALTER TABLE tcrdb.clone_responses ADD cdna_id int; +ALTER TABLE tcrdb.clone_responses ADD nostimid int; +ALTER TABLE tcrdb.clone_responses ADD chain varchar(100); +ALTER TABLE tcrdb.clone_responses ADD clonotype varchar(1000); + diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.53-15.54.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.53-15.54.sql new file mode 100644 index 000000000..54362b3cc --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.53-15.54.sql @@ -0,0 +1,20 @@ +CREATE TABLE tcrdb.stims ( + rowid int identity(1,1), + cdna_id int, + controlStimId int, + + quantificationMethod varchar(1000), + quantification double precision, + + flowQuantificationMethod varchar(1000), + flowQuantification double precision, + comment varchar(4000), + + container entityid, + created datetime, + createdby int, + modified datetime, + modifiedby int, + + constraint PK_stims PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index a6dd79ed1..88951dd56 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -97,476 +97,36 @@ - +
- TCR Stims/Samples - rowid - - - Stim Id - - - Tube # - - - Animal/Cell - http://cpas.labkey.com/Study#ParticipantId - false - - - Tissue - true - - - Effectors - false - - - # Effectors - - - APCs - true - - - # APCs - true - - - Sample Date - http://cpas.labkey.com/laboratory#sampleDate - false - Date - - - Peptide/Stim - false - - tcrdb - peptides - stim - - - - - - - - Co-stim - - - Background Freq - - - Activated Freq - - - - - - - - - lsidtype - true - true - false - - ObjectUri - Object - exp - - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - - true - - - TCR Sorts + TCR Clone Responsesrowid - Sort Id - false - - - Stim Id - - tcrdb - stims - rowid - - false - - - false - - - - - - # Cells - false - - - true - - - false - - laboratory - 96well_plate - well - - - - - Hashtag Oligo - - tcrdb - hashtag_oligos - tag_name - - - - - - - - - - - lsidtype - true - true - false - - ObjectUri - Object - exp - - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 + Row Id true - -
- - - TCR cDNA Libraries - rowid - - - Library Id - false - - - Sort Id - - tcrdb - sorts - rowid - - false - - - - - - - - - false - - - false - - laboratory - 96well_plate - well - - - - - Readset (GEX) - - sequenceanalysis - sequence_readsets - rowid - - - - Readset (TCR) - - sequenceanalysis - sequence_readsets - rowid - - - - Readset (Cell Hashing) + + cDNA ID - sequenceanalysis - sequence_readsets + singlecell + cdna_libraries rowid - - Readset (CITE-Seq) + + No Stim / Control cDNA ID - sequenceanalysis - sequence_readsets + singlecell + cdna_libraries rowid - - cDNA Concentration (ng/uL) - - - TCR Enriched Concentration (ng/uL) - - - Cite-Seq Panel - - tcrdb - citeseq_panel_names - name - - - - - - - - - - - lsidtype - true - true - false - - ObjectUri - Object - exp - - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - - - laboratory/buttons.js - - - - Laboratory.buttonHandlers.importDataHandlerForView(dataRegionName, 'tcrdb', 'stimDashboard') - - - Laboratory.buttonHandlers.importDataHandlerForView(dataRegionName, 'tcrdb', 'poolImport') - - - - - - -
- - - Peptides/Stims - - - Peptide/Stim - - - Category - - - Type - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - Processing Needed/Plate - - - Row Id - - - Plate Id - - - Processing Needed - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - - TCR Clone Responses - rowid - - - Row Id - true - - - Experiment - - - Clone Name - true - - - Cell Background - - - Date - http://cpas.labkey.com/laboratory#sampleDate - Date - - - # Effectors - - - Pct Transduction - - - Peptide/Stim - false - - - Co-stim + + Chain - - Antigen + + Clonotype Activated Freq @@ -685,8 +245,6 @@ false false false - true - 29 true @@ -696,101 +254,50 @@ false false false - true - 29 true
- - - CITE-Seq Panels + +
+ + Stim Experiments + rowid Row Id true - - Panel Name - - - Marker/Antigen + + cDNA ID - tcrdb - citeseq_antibodies - antibodyName - + singlecell + cdna_libraries + rowid - - Marker/Antigen Label In Panel - If the desired label is different than the default label for this marker, enter a value - - - true - - - true - - - false - false - false - true - 29 - true - - - true - - - false - false - false - true - 29 - true - - -
- - - CITE-Seq Antibodies - - - Row Id - true - - - Antibody Name - - - Marker/Antigen Name - - - Marker/Antigen Label + + No Stim / Control cDNA ID + + singlecell + cdna_libraries + rowid + - - Clone Name + + Quantification Method - - Vendor + + Quantification - - Product Id + + Flow Quantification Method - - Barcode Name - - sequenceanalysis - barcodes - tag_name - - + + Flow Quantification - - Adapter Sequence - textarea + + true @@ -803,7 +310,6 @@ false false true - 29 true @@ -814,7 +320,6 @@ false false true - 29 true diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 6d5f552f6..dd7b9d912 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -46,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.52; + return 15.54; } @Override diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java index ebde29466..6af64dd6d 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java @@ -69,6 +69,9 @@ public List getDataNavItems(Container c, User u) } items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, "TCR Clones", LaboratoryService.NavItemCategory.data, NAME, cache)); + items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONE_RESPONSES, "TCR Clonotype Responses", LaboratoryService.NavItemCategory.data, NAME, cache)); + items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIM_EXPERIMENTS, "T-cell Stim Experiments", LaboratoryService.NavItemCategory.data, NAME, cache)); + return Collections.unmodifiableList(items); } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java index 32a0ca98f..5bafb8f0c 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java @@ -27,6 +27,8 @@ public class TCRdbSchema public static final String TABLE_MIXCR_LIBRARIES = "mixcr_libraries"; public static final String TABLE_CLONES = "clones"; + public static final String TABLE_CLONE_RESPONSES = "clone_responses"; + public static final String TABLE_STIM_EXPERIMENTS = "stims"; public static final String SEQUENCE_ANALYSIS = "sequenceanalysis"; public static final String SINGLE_CELL = "singlecell";