From 9a9a8d06e80566a9ca040cb2ed8d7d43caa6a886 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 4 Sep 2023 12:05:16 -0700 Subject: [PATCH 1/5] Add action to allow external server to trigger re-import of externally computed kinship data --- GeneticsCore/build.gradle | 2 + .../resources/etls/KinshipDataImport.xml | 14 +++ GeneticsCore/resources/module.xml | 7 ++ .../GeneticsCore/GeneticsCoreController.java | 27 +++++ .../etl/ImportGeneticsCalculationsStep.java | 107 ++++++++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 GeneticsCore/resources/etls/KinshipDataImport.xml create mode 100644 GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java diff --git a/GeneticsCore/build.gradle b/GeneticsCore/build.gradle index 3d96f0f6f..0b773f12f 100644 --- a/GeneticsCore/build.gradle +++ b/GeneticsCore/build.gradle @@ -6,6 +6,8 @@ plugins { dependencies { BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: 'published', depExtension: 'module') BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: 'published', depExtension: 'module') BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") diff --git a/GeneticsCore/resources/etls/KinshipDataImport.xml b/GeneticsCore/resources/etls/KinshipDataImport.xml new file mode 100644 index 000000000..eb666d9db --- /dev/null +++ b/GeneticsCore/resources/etls/KinshipDataImport.xml @@ -0,0 +1,14 @@ + + + KinshipDataImport + Import PRIMe-seq Kinship Data + + + + + + + + + + diff --git a/GeneticsCore/resources/module.xml b/GeneticsCore/resources/module.xml index cf9c91b9a..15a6c54a2 100644 --- a/GeneticsCore/resources/module.xml +++ b/GeneticsCore/resources/module.xml @@ -21,6 +21,13 @@ This is the containerPath to the folder holding the primary data for Parentage. Use of slashes is very important - it should be in the format '/myProject/folder' + + true + + ADMIN + + This is the filepath of a folder where externally kinship/inbreeding TSVs calculated externally will be deposited. It is used by the KinshipData ETL to initiate import. + diff --git a/GeneticsCore/src/org/labkey/GeneticsCore/GeneticsCoreController.java b/GeneticsCore/src/org/labkey/GeneticsCore/GeneticsCoreController.java index d1b201e96..77f3e6494 100644 --- a/GeneticsCore/src/org/labkey/GeneticsCore/GeneticsCoreController.java +++ b/GeneticsCore/src/org/labkey/GeneticsCore/GeneticsCoreController.java @@ -4,10 +4,15 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.labkey.GeneticsCore.mhc.MhcTaskRef; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.MutatingApiAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.di.DataIntegrationService; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.util.HtmlString; import org.labkey.api.util.URLHelper; import org.labkey.api.view.HtmlView; @@ -62,4 +67,26 @@ public URLHelper getSuccessURL(Object o) return getContainer().getStartURL(getUser()); } } + + + @RequiresPermission(UpdatePermission.class) + public static class ImportGeneticsDataAction extends MutatingApiAction + { + @Override + public ApiResponse execute(Object form, BindException errors) + { + try + { + DataIntegrationService.get().runTransformNow(getContainer(), getUser(), "{GeneticsCore}/KinshipDataImport"); + + return new ApiSimpleResponse("success", true); + } + catch (Exception e) + { + _log.error("Unable to initiate genetics data import", e); + errors.reject(ERROR_MSG, "Unable to initiate genetics data import"); + return null; + } + } + } } diff --git a/GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java b/GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java new file mode 100644 index 000000000..d0515affa --- /dev/null +++ b/GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java @@ -0,0 +1,107 @@ +package org.labkey.GeneticsCore.etl; + +import org.apache.commons.lang3.StringUtils; +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.GeneticsCore.GeneticsCoreModule; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.ehr.EHRService; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.ContainerUser; + +import java.io.File; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ImportGeneticsCalculationsStep implements TaskRefTask +{ + protected ContainerUser _containerUser; + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + Module ehr = ModuleLoader.getInstance().getModule("ehr"); + Module geneticsCore = ModuleLoader.getInstance().getModule(GeneticsCoreModule.class); + + ModuleProperty mp = ehr.getModuleProperties().get("EHRStudyContainer"); + String ehrContainerPath = StringUtils.trimToNull(mp.getEffectiveValue(_containerUser.getContainer())); + if (ehrContainerPath == null) + { + throw new PipelineJobException("EHRStudyContainer has not been set"); + } + + Container ehrContainer = ContainerManager.getForPath(ehrContainerPath); + if (ehrContainer == null) + { + throw new PipelineJobException("Invalid container: " + ehrContainerPath); + } + + if (!_containerUser.getContainer().equals(ehrContainer)) + { + throw new PipelineJobException("This ETL can only be run from the EHRStudyContainer"); + } + + // Downstream import events will get additional permissions checks + if (!ehrContainer.hasPermission(_containerUser.getUser(), UpdatePermission.class)) + { + throw new UnauthorizedException(); + } + + ModuleProperty mp2 = geneticsCore.getModuleProperties().get("KinshipDataPath"); + String pipeDirPath = StringUtils.trimToNull(mp2.getEffectiveValue(ehrContainer)); + if (pipeDirPath == null) + { + throw new PipelineJobException("Must provide the filepath to import data using the KinshipDataPath module property"); + } + + File pipeDir = new File(pipeDirPath); + if (!pipeDir.exists()) + { + throw new PipelineJobException("Path does not exist: " + pipeDir.getPath()); + } + + File kinship = new File(pipeDir, "kinship.txt"); + if (!kinship.exists()) + { + throw new PipelineJobException("File does not exist: " + kinship.getPath()); + } + + File inbreeding = new File(pipeDir, "inbreeding.txt"); + if (!inbreeding.exists()) + { + throw new PipelineJobException("File does not exist: " + inbreeding.getPath()); + } + + EHRService.get().standaloneProcessKinshipAndInbreeding(ehrContainer, _containerUser.getUser(), pipeDir, job.getLogger()); + + return new RecordedActionSet(); + } + + @Override + public List getRequiredSettings() + { + return Collections.emptyList(); + } + + @Override + public void setSettings(Map settings) throws XmlException + { + + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} From a228e51a07b9e719ef8f7c5daccaf480e0b6851a Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 6 Sep 2023 06:08:36 -0700 Subject: [PATCH 2/5] Try to fix unrelated ONPRC_EHRTest2 failure --- .../src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java b/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java index 7c3e6e2de..8dca34207 100644 --- a/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java +++ b/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java @@ -1193,7 +1193,9 @@ private void updateTotalVolume(Integer quantity) private void addProjectToTheRow(Ext4GridRef gridRef, int index, String project) { - gridRef.clickDownArrowOnGrid(index, "project"); + gridRef.startEditing(index, "project"); + Ext4FieldRef fieldRef = gridRef.getActiveEditor(index, "project"); + fieldRef.eval("expand()"); waitAndClick(Locator.tag("li").append(Locator.tagContainingText("span", "Other"))); waitForElement(Ext4Helper.Locators.window("Choose Project")); _ext4Helper.queryOne("window[title=Choose Project] [fieldLabel='Project']", Ext4ComboRef.class).setComboByDisplayValue(project); From 07b780caa408e3d58b6a0d494433a0b88a576bef Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 12 Sep 2023 11:33:23 -0700 Subject: [PATCH 3/5] Add setting to control whether kinship import is allowed during business hours --- .../test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest.java b/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest.java index 7763fcd88..ccc1beab8 100644 --- a/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest.java +++ b/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest.java @@ -1261,6 +1261,7 @@ public void testGeneticsPipeline() throws Exception waitAndClickAndWait(Locators.bodyPanel().append(Locator.tagContainingText("a", "EHR Admin Page"))); waitAndClickAndWait(Locator.tagContainingText("a", "Genetics Calculations")); _ext4Helper.checkCheckbox(Ext4Helper.Locators.checkbox(this, "Kinship validation?:")); + _ext4Helper.checkCheckbox(Ext4Helper.Locators.checkbox(this, "Allow Import During Business Hours?:")); Locator loc = Locator.inputByIdContaining("numberfield"); waitForElement(loc); setFormElement(loc, "23"); From 081ca38d33eea4b7b7ef2a68cd763e8de0243dc1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 28 Sep 2023 10:27:00 -0700 Subject: [PATCH 4/5] VetAssignmentAndCacheFalsePositives --- onprc_ehr/resources/queries/onprc_ehr/vet_assignment.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/onprc_ehr/resources/queries/onprc_ehr/vet_assignment.js b/onprc_ehr/resources/queries/onprc_ehr/vet_assignment.js index d18819474..0e9759109 100644 --- a/onprc_ehr/resources/queries/onprc_ehr/vet_assignment.js +++ b/onprc_ehr/resources/queries/onprc_ehr/vet_assignment.js @@ -1,10 +1,8 @@ require("ehr/triggers").initScript(this); var triggerHelper = new org.labkey.onprc_ehr.query.ONPRC_EHRTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id); -var console = require("console"); EHR.Server.TriggerManager.registerHandlerForQuery(EHR.Server.TriggerManager.Events.COMPLETE, 'onprc_ehr', 'vet_assignment', function(event, helper){ - console.log('Complete is called!'); // NOTE: the rules behind vet assignment are complicated enough that any change to one row here could // impact a lot of records. Therefore, just redo everything in the cache triggerHelper.recalculateAllVetAssignmentRecords() From 50987dae5716ce01bd9d8266e0a82c1937d02a84 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 28 Feb 2024 09:25:55 -0800 Subject: [PATCH 5/5] Back out some changes not directly related to remote kinship execution --- .../src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java b/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java index 8dca34207..7c3e6e2de 100644 --- a/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java +++ b/onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java @@ -1193,9 +1193,7 @@ private void updateTotalVolume(Integer quantity) private void addProjectToTheRow(Ext4GridRef gridRef, int index, String project) { - gridRef.startEditing(index, "project"); - Ext4FieldRef fieldRef = gridRef.getActiveEditor(index, "project"); - fieldRef.eval("expand()"); + gridRef.clickDownArrowOnGrid(index, "project"); waitAndClick(Locator.tag("li").append(Locator.tagContainingText("span", "Other"))); waitForElement(Ext4Helper.Locators.window("Choose Project")); _ext4Helper.queryOne("window[title=Choose Project] [fieldLabel='Project']", Ext4ComboRef.class).setComboByDisplayValue(project);