From 7235b13a62832be9e3c64e4b59e2f85e0472f02e Mon Sep 17 00:00:00 2001 From: Lum Date: Tue, 17 Dec 2024 16:16:50 -0800 Subject: [PATCH 1/2] repackage/clean up transform script code introduce AnalysisScript class to handle different scripting operations --- .../api/assay/AbstractAssayProvider.java | 49 +- .../org/labkey/api/assay/AssayProvider.java | 13 +- .../api/assay/AssayRunDatabaseContext.java | 4 +- .../api/assay/AssayRunUploadContext.java | 2 +- .../api/assay/AssayRunUploadContextImpl.java | 5 +- .../api/assay/DefaultAssayRunCreator.java | 9 +- .../org/labkey/api/assay/TsvDataHandler.java | 3 +- .../api/assay/actions/AssayRunUploadForm.java | 4 +- .../assay/pipeline/AssayRunAsyncContext.java | 4 +- .../api/assay/transform/AnalysisScript.java | 96 +++ .../transform}/DataExchangeHandler.java | 121 ++-- .../DataTransformService.java} | 626 +++++++++--------- .../transform}/DefaultTransformResult.java | 322 +++++---- .../transform}/TransformDataHandler.java | 67 +- .../transform}/TransformResult.java | 151 ++--- .../org/labkey/api/qc/DataTransformer.java | 34 - .../labkey/api/qc/TsvDataExchangeHandler.java | 4 +- .../org/labkey/api/qc/TsvDataSerializer.java | 1 + .../report/ExternalScriptEngineReport.java | 4 +- .../AbstractDilutionAssayProvider.java | 2 +- .../src/org/labkey/assay/AssayController.java | 2 +- .../labkey/assay/AssayDomainServiceImpl.java | 20 +- assay/src/org/labkey/assay/AssayModule.java | 4 +- .../org/labkey/assay/ModuleAssayProvider.java | 28 +- .../org/labkey/assay/TsvAssayProvider.java | 2 +- .../pipeline/analysis/CommandTaskImpl.java | 6 +- .../pipeline/api/SimpleTaskFactory.java | 8 +- .../pipeline/FileAnalysisDatasetTask.java | 6 +- 28 files changed, 802 insertions(+), 795 deletions(-) create mode 100644 api/src/org/labkey/api/assay/transform/AnalysisScript.java rename api/src/org/labkey/api/{qc => assay/transform}/DataExchangeHandler.java (95%) rename api/src/org/labkey/api/assay/{DefaultDataTransformer.java => transform/DataTransformService.java} (78%) rename api/src/org/labkey/api/{qc => assay/transform}/DefaultTransformResult.java (88%) rename api/src/org/labkey/api/{qc => assay/transform}/TransformDataHandler.java (91%) rename api/src/org/labkey/api/{qc => assay/transform}/TransformResult.java (65%) delete mode 100644 api/src/org/labkey/api/qc/DataTransformer.java diff --git a/api/src/org/labkey/api/assay/AbstractAssayProvider.java b/api/src/org/labkey/api/assay/AbstractAssayProvider.java index 0a138def90d..c5229f71fb0 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayProvider.java +++ b/api/src/org/labkey/api/assay/AbstractAssayProvider.java @@ -20,11 +20,15 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; import org.labkey.api.assay.actions.AssayRunUploadForm; import org.labkey.api.assay.actions.DesignerAction; import org.labkey.api.assay.actions.UploadWizardAction; import org.labkey.api.assay.pipeline.AssayRunAsyncContext; import org.labkey.api.assay.security.DesignAssayPermission; +import org.labkey.api.assay.transform.AnalysisScript; +import org.labkey.api.assay.transform.DataExchangeHandler; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.audit.AuditLogService; import org.labkey.api.data.ActionButton; import org.labkey.api.data.ButtonBar; @@ -78,7 +82,6 @@ import org.labkey.api.module.Module; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.qc.DataExchangeHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.query.FilteredTable; import org.labkey.api.query.QueryService; @@ -1232,16 +1235,15 @@ public boolean hasUsefulDetailsPage() private static final String SCRIPT_PATH_DELIMITER = "|"; @Override - public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List scripts) throws ExperimentException + public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List scripts) throws ExperimentException { Map props = new HashMap<>(protocol.getObjectProperties()); String propertyURI = ScriptType.TRANSFORM.getPropertyURI(protocol); - ValidationException validationErrors = new ValidationException(); - StringBuilder sb = new StringBuilder(); - String separator = ""; - for (File scriptFile : scripts) + + for (AnalysisScript script : scripts) { + File scriptFile = script.getScript().toNioPathForRead().toFile(); String ext = FileUtil.getExtension(scriptFile); if (scriptFile.isFile() && ext != null) { @@ -1263,10 +1265,6 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, validationErrors.addErrors(ParamReplacementSvc.get().validateDeprecatedReplacements(scriptText, scriptFile.getName())); } - - sb.append(separator); - sb.append(scriptFile.getAbsolutePath()); - separator = SCRIPT_PATH_DELIMITER; } else { @@ -1283,20 +1281,17 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, if (validationErrors.getErrors().stream().anyMatch(e -> SEVERITY.ERROR == e.getSeverity())) return validationErrors; - if (sb.length() > 0) + JSONArray json = AnalysisScript.toJson(scripts); + if (json != null) { ObjectProperty prop = new ObjectProperty(protocol.getLSID(), protocol.getContainer(), - propertyURI, sb.toString()); + propertyURI, json.toString()); props.put(propertyURI, prop); } else { props.remove(propertyURI); } - - // Be sure to strip out any validation scripts that were stored with the legacy propertyURI. We merge and save - // them as a single list in the TRANSFORM - props.remove(ScriptType.VALIDATION.getPropertyURI(protocol)); protocol.setObjectProperties(props); return validationErrors; @@ -1305,7 +1300,6 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, /** For migrating legacy assay designs that have separate transform and validation script properties */ private enum ScriptType { - VALIDATION("ValidationScript"), TRANSFORM("TransformScript"); private final String _uriSuffix; @@ -1323,26 +1317,21 @@ public String getPropertyURI(ExpProtocol protocol) @NotNull @Override - public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope) + public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope) { - List result = new ArrayList<>(); + List result = new ArrayList<>(); if (scope == Scope.ASSAY_DEF || scope == Scope.ALL) { ObjectProperty transformScripts = protocol.getObjectProperties().get(ScriptType.TRANSFORM.getPropertyURI(protocol)); if (transformScripts != null) { + List scripts = AnalysisScript.fromJson(transformScripts.getStringValue()); + if (scripts != null) + return scripts; + + // try the legacy serialization for (String scriptPath : transformScripts.getStringValue().split("\\" + SCRIPT_PATH_DELIMITER)) - { - result.add(new File(scriptPath)); - } - } - ObjectProperty validationScripts = protocol.getObjectProperties().get(ScriptType.VALIDATION.getPropertyURI(protocol)); - if (validationScripts != null) - { - for (String scriptPath : validationScripts.getStringValue().split("\\" + SCRIPT_PATH_DELIMITER)) - { - result.add(new File(scriptPath)); - } + result.add(new AnalysisScript(new File(scriptPath), Set.of(DataTransformService.TransformOperation.INSERT))); } } return result; diff --git a/api/src/org/labkey/api/assay/AssayProvider.java b/api/src/org/labkey/api/assay/AssayProvider.java index b3c912ab681..5f7b7ca11f6 100644 --- a/api/src/org/labkey/api/assay/AssayProvider.java +++ b/api/src/org/labkey/api/assay/AssayProvider.java @@ -16,11 +16,14 @@ package org.labkey.api.assay; +import jakarta.servlet.http.HttpServletRequest; import org.fhcrc.cpas.exp.xml.ExperimentRunType; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.actions.AssayRunUploadForm; import org.labkey.api.assay.pipeline.AssayRunAsyncContext; +import org.labkey.api.assay.transform.AnalysisScript; +import org.labkey.api.assay.transform.DataExchangeHandler; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.exp.ExperimentException; @@ -40,11 +43,10 @@ import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; import org.labkey.api.module.Module; import org.labkey.api.pipeline.PipelineProvider; -import org.labkey.api.qc.DataExchangeHandler; import org.labkey.api.query.ValidationException; import org.labkey.api.security.User; -import org.labkey.api.study.publish.PublishKey; import org.labkey.api.study.assay.ParticipantVisitResolverType; +import org.labkey.api.study.publish.PublishKey; import org.labkey.api.util.Pair; import org.labkey.api.view.ActionURL; import org.labkey.api.view.HttpView; @@ -55,7 +57,6 @@ import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.mvc.Controller; -import jakarta.servlet.http.HttpServletRequest; import java.io.File; import java.io.IOException; import java.util.Collection; @@ -236,12 +237,10 @@ enum Scope * File based QC and analysis scripts can be added to a protocol and invoked when the validate * method is called. Set to an empty list if no scripts exist. */ - // TODO File->FileLike - ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List scripts) throws ExperimentException; + ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List scripts) throws ExperimentException; @NotNull - // TODO File->FileLike - List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope); + List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope); void setSaveScriptFiles(ExpProtocol protocol, boolean save) throws ExperimentException; boolean isSaveScriptFiles(ExpProtocol protocol); diff --git a/api/src/org/labkey/api/assay/AssayRunDatabaseContext.java b/api/src/org/labkey/api/assay/AssayRunDatabaseContext.java index 86bf45734d9..70cc88dcf93 100644 --- a/api/src/org/labkey/api/assay/AssayRunDatabaseContext.java +++ b/api/src/org/labkey/api/assay/AssayRunDatabaseContext.java @@ -29,8 +29,8 @@ import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.qc.DefaultTransformResult; -import org.labkey.api.qc.TransformResult; +import org.labkey.api.assay.transform.DefaultTransformResult; +import org.labkey.api.assay.transform.TransformResult; import org.labkey.api.security.User; import org.labkey.api.util.NetworkDrive; import org.labkey.api.view.ActionURL; diff --git a/api/src/org/labkey/api/assay/AssayRunUploadContext.java b/api/src/org/labkey/api/assay/AssayRunUploadContext.java index c525fbc332d..4c65940801c 100644 --- a/api/src/org/labkey/api/assay/AssayRunUploadContext.java +++ b/api/src/org/labkey/api/assay/AssayRunUploadContext.java @@ -27,7 +27,7 @@ import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.qc.TransformResult; +import org.labkey.api.assay.transform.TransformResult; import org.labkey.api.security.User; import org.labkey.api.view.ActionURL; import org.labkey.api.view.HasHttpRequest; diff --git a/api/src/org/labkey/api/assay/AssayRunUploadContextImpl.java b/api/src/org/labkey/api/assay/AssayRunUploadContextImpl.java index 428fa9da490..972c0b7c230 100644 --- a/api/src/org/labkey/api/assay/AssayRunUploadContextImpl.java +++ b/api/src/org/labkey/api/assay/AssayRunUploadContextImpl.java @@ -30,8 +30,8 @@ import org.labkey.api.exp.api.ExperimentJSONConverter; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.qc.DefaultTransformResult; -import org.labkey.api.qc.TransformResult; +import org.labkey.api.assay.transform.DefaultTransformResult; +import org.labkey.api.assay.transform.TransformResult; import org.labkey.api.security.User; import org.labkey.api.util.URIUtil; import org.labkey.api.view.ActionURL; @@ -39,7 +39,6 @@ import org.labkey.api.view.ViewContext; import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.HashMap; diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index 78b5511f7fc..91232d21ea2 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -27,6 +27,9 @@ import org.labkey.api.assay.pipeline.AssayRunAsyncContext; import org.labkey.api.assay.pipeline.AssayUploadPipelineJob; import org.labkey.api.assay.sample.AssaySampleLookupContext; +import org.labkey.api.assay.transform.DataTransformService; +import org.labkey.api.assay.transform.TransformDataHandler; +import org.labkey.api.assay.transform.TransformResult; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -64,9 +67,6 @@ import org.labkey.api.exp.property.ValidatorContext; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineValidationException; -import org.labkey.api.qc.DataTransformer; -import org.labkey.api.qc.TransformDataHandler; -import org.labkey.api.qc.TransformResult; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.PropertyValidationError; import org.labkey.api.query.SimpleValidationError; @@ -113,8 +113,7 @@ public DefaultAssayRunCreator(ProviderType provider) public TransformResult transform(AssayRunUploadContext context, ExpRun run) throws ValidationException { - DataTransformer transformer = new DefaultDataTransformer<>(); - return transformer.transformAndValidate(context, run); + return DataTransformService.get().transformAndValidate(context, run, DataTransformService.TransformOperation.INSERT); } @Override diff --git a/api/src/org/labkey/api/assay/TsvDataHandler.java b/api/src/org/labkey/api/assay/TsvDataHandler.java index 869c16d1a18..52bbc6a1b67 100644 --- a/api/src/org/labkey/api/assay/TsvDataHandler.java +++ b/api/src/org/labkey/api/assay/TsvDataHandler.java @@ -28,7 +28,6 @@ import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Sort; import org.labkey.api.data.TSVGridWriter; -import org.labkey.api.data.TSVWriter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.exp.ExperimentException; @@ -38,7 +37,7 @@ import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.qc.TransformDataHandler; +import org.labkey.api.assay.transform.TransformDataHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.security.User; import org.labkey.api.util.FileType; diff --git a/api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java b/api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java index 65be3513c69..7f227d8181d 100644 --- a/api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java +++ b/api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java @@ -49,8 +49,8 @@ import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.qc.DefaultTransformResult; -import org.labkey.api.qc.TransformResult; +import org.labkey.api.assay.transform.DefaultTransformResult; +import org.labkey.api.assay.transform.TransformResult; import org.labkey.api.qc.TsvDataExchangeHandler; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.PdLookupForeignKey; diff --git a/api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java b/api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java index 540d632382d..eaa4369ebc2 100644 --- a/api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java +++ b/api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java @@ -31,8 +31,8 @@ import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.qc.DefaultTransformResult; -import org.labkey.api.qc.TransformResult; +import org.labkey.api.assay.transform.DefaultTransformResult; +import org.labkey.api.assay.transform.TransformResult; import org.labkey.api.security.User; import org.labkey.api.security.UserManager; import org.labkey.api.view.ActionURL; diff --git a/api/src/org/labkey/api/assay/transform/AnalysisScript.java b/api/src/org/labkey/api/assay/transform/AnalysisScript.java new file mode 100644 index 00000000000..a4f3134606c --- /dev/null +++ b/api/src/org/labkey/api/assay/transform/AnalysisScript.java @@ -0,0 +1,96 @@ +package org.labkey.api.assay.transform; + +import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class AnalysisScript +{ + FileLike _script; + Set _operations = new HashSet<>(); + + public AnalysisScript(File script, Set operations) + { + _script = new FileSystemLike.Builder(script).build().getRoot(); + _operations = operations; + } + + private AnalysisScript(File script, List operations) + { + _script = new FileSystemLike.Builder(script).build().getRoot(); + for (String op : operations) + _operations.add(DataTransformService.TransformOperation.valueOf(op)); + } + + public FileLike getScript() + { + return _script; + } + + public boolean canExecute(DataTransformService.TransformOperation operation) + { + return _operations.contains(operation); + } + + private static JSONObject toJson(AnalysisScript script) + { + JSONObject json = new JSONObject(); + + json.put("operations", script._operations); + json.put("script", script._script.toNioPathForRead()); + + return json; + } + + @Nullable + public static JSONArray toJson(Collection scripts) + { + if (!scripts.isEmpty()) + { + JSONArray json = new JSONArray(); + for (AnalysisScript script : scripts) + { + json.put(toJson(script)); + } + return json; + } + return null; + } + + @Nullable + public static List fromJson(String jsonStr) + { + try + { + List scripts = new ArrayList<>(); + JSONArray json = new JSONArray(jsonStr); + for (Object o : json.toList()) + { + if (o instanceof Map props) + { + File script = new File(String.valueOf(props.get("script"))); + List ops = (List) props.get("operations"); + + scripts.add(new AnalysisScript(script, ops)); + } + } + return scripts; + } + catch (JSONException e) + { + // ignore + return null; + } + } +} \ No newline at end of file diff --git a/api/src/org/labkey/api/qc/DataExchangeHandler.java b/api/src/org/labkey/api/assay/transform/DataExchangeHandler.java similarity index 95% rename from api/src/org/labkey/api/qc/DataExchangeHandler.java rename to api/src/org/labkey/api/assay/transform/DataExchangeHandler.java index ebaf1a811da..539d5af9af2 100644 --- a/api/src/org/labkey/api/qc/DataExchangeHandler.java +++ b/api/src/org/labkey/api/assay/transform/DataExchangeHandler.java @@ -1,62 +1,61 @@ -/* - * Copyright (c) 2009-2016 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.qc; - -import org.jetbrains.annotations.NotNull; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayRunUploadContext; -import org.labkey.api.data.TSVWriter; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.ValidationException; -import org.labkey.api.util.Pair; -import org.labkey.api.view.ViewContext; -import org.labkey.vfs.FileLike; - -import java.io.File; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Used to process input and output data between the server and externally executed qc and analysis scripts. - * User: Karl Lum - * Date: Jan 7, 2009 - */ -public interface DataExchangeHandler -{ - Pair> createTransformationRunInfo(AssayRunUploadContext context, ExpRun run, FileLike scriptDir, Map runProperties, Map batchProperties) throws Exception; - void createSampleData(@NotNull ExpProtocol protocol, ViewContext viewContext, FileLike scriptDir) throws Exception; - - TransformResult processTransformationOutput(AssayRunUploadContext context, FileLike runInfo, ExpRun run, FileLike scriptFile, TransformResult mergeResult, Set inputDataFiles) throws ValidationException; - - DataSerializer getDataSerializer(); - - interface DataSerializer - { - /** - * Called to save or import transformed or QC'd run data to the specified reader or writer. - */ - void exportRunData(ExpProtocol protocol, List data, FileLike runData, TSVWriter tsvWriter) throws IOException, BatchValidationException; - - DataIteratorBuilder importRunData(ExpProtocol protocol, File runData) throws Exception; - } +/* + * Copyright (c) 2009-2016 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.assay.transform; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayRunUploadContext; +import org.labkey.api.data.TSVWriter; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.ValidationException; +import org.labkey.api.util.Pair; +import org.labkey.api.view.ViewContext; +import org.labkey.vfs.FileLike; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Used to process input and output data between the server and externally executed qc and analysis scripts. + * User: Karl Lum + * Date: Jan 7, 2009 + */ +public interface DataExchangeHandler +{ + Pair> createTransformationRunInfo(AssayRunUploadContext context, ExpRun run, FileLike scriptDir, Map runProperties, Map batchProperties) throws Exception; + void createSampleData(@NotNull ExpProtocol protocol, ViewContext viewContext, FileLike scriptDir) throws Exception; + + TransformResult processTransformationOutput(AssayRunUploadContext context, FileLike runInfo, ExpRun run, FileLike scriptFile, TransformResult mergeResult, Set inputDataFiles) throws ValidationException; + + DataSerializer getDataSerializer(); + + interface DataSerializer + { + /** + * Called to save or import transformed or QC'd run data to the specified reader or writer. + */ + void exportRunData(ExpProtocol protocol, List data, FileLike runData, TSVWriter tsvWriter) throws IOException, BatchValidationException; + + DataIteratorBuilder importRunData(ExpProtocol protocol, File runData) throws Exception; + } } \ No newline at end of file diff --git a/api/src/org/labkey/api/assay/DefaultDataTransformer.java b/api/src/org/labkey/api/assay/transform/DataTransformService.java similarity index 78% rename from api/src/org/labkey/api/assay/DefaultDataTransformer.java rename to api/src/org/labkey/api/assay/transform/DataTransformService.java index 9d0456c4461..90c0be97e77 100644 --- a/api/src/org/labkey/api/assay/DefaultDataTransformer.java +++ b/api/src/org/labkey/api/assay/transform/DataTransformService.java @@ -1,317 +1,309 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.assay; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobService; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.qc.DataExchangeHandler; -import org.labkey.api.qc.DataTransformer; -import org.labkey.api.qc.DefaultTransformResult; -import org.labkey.api.qc.TransformResult; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reader.Readers; -import org.labkey.api.reports.ExternalScriptEngine; -import org.labkey.api.reports.LabKeyScriptEngineManager; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityManager.TransformSession; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.CSRFUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; - -import javax.script.Bindings; -import javax.script.ScriptContext; -import javax.script.ScriptEngine; -import jakarta.servlet.http.HttpServletRequest; -import org.labkey.api.util.UnexpectedException; -import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * Runs the selected data file(s) through the assay design's transform scripts. Writes the data to a standard TSV - * format when possible to make it easier for the scripts to consume. Loads the script's output into a TransformResult - * so it can be consumed/imported by other code. - * - * User: klum - * Date: Sep 22, 2009 - */ -public class DefaultDataTransformer implements DataTransformer -{ - public static final String RUN_INFO_REPLACEMENT = "runInfo"; - public static final String SRC_DIR_REPLACEMENT = "srcDirectory"; - public static final String R_SESSIONID_REPLACEMENT = "rLabkeySessionId"; - public static final String LEGACY_SESSION_ID_REPLACEMENT = "httpSessionId"; - public static final String LEGACY_SESSION_COOKIE_NAME_REPLACEMENT = "sessionCookieName"; - public static final String BASE_SERVER_URL_REPLACEMENT = "baseServerURL"; - public static final String CONTAINER_PATH = "containerPath"; - public static final String ORIGINAL_SOURCE_PATH = "OriginalSourcePath"; - - @Override - public TransformResult transformAndValidate(AssayRunUploadContext context, ExpRun run) throws ValidationException - { - boolean isDefault = isDefault(context.getProtocol()); - TransformResult result = DefaultTransformResult.createEmptyResult(); - DataExchangeHandler dataHandler = context.getProvider().createDataExchangeHandler(); - if (dataHandler == null) - { - return result; - } - - Map batchProperties; - Map runProperties; - try - { - batchProperties = context.getBatchProperties(); - runProperties = context.getRunProperties(); - } - catch (ExperimentException e) - { - throw new ValidationException(e.getMessage() == null ? e.toString() : e.getMessage()); - } - - for (File scriptFile : context.getProvider().getValidationAndAnalysisScripts(context.getProtocol(), AssayProvider.Scope.ALL)) - { - // read the contents of the script file - if (scriptFile.exists()) - { - // TODO: don't bother slurping the contents of binary scripts - StringBuilder sb = new StringBuilder(); - try (BufferedReader br = Readers.getReader(scriptFile)) - { - String l; - while ((l = br.readLine()) != null) - sb.append(l).append('\n'); - } - catch (Exception e) - { - throw new ValidationException(e.getMessage()); - } - ScriptEngine engine = null; - String ext = FileUtil.getExtension(scriptFile); - if (ext != null) - { - engine = LabKeyScriptEngineManager.get() - .getEngineByExtension(context.getContainer(), ext, LabKeyScriptEngineManager.EngineContext.pipeline); - } - if (engine != null) - { - // issue : 46838 remote scripting engines don't support transform scripts yet - if (engine instanceof ExternalScriptEngine externalScriptEngine) - { - if (!externalScriptEngine.supportsContext(LabKeyScriptEngineManager.EngineContext.pipeline)) - throw new ValidationException("The script engine : " + externalScriptEngine.getEngineDefinition().getName() + " does not support running in a transform script." ); - } - FileLike scriptDir = null; - // issue 19748: need alternative to JSESSIONID for pipeline job transform script usage (i.e., TransformSession) - try (TransformSession session = SecurityManager.createTransformSession(context)) - { - scriptDir = getScriptDir(context.getProtocol(), scriptFile, isDefault); - // issue 13643: ensure script dir is initially empty - FileUtil.deleteDirectoryContents(scriptDir); - - Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); - String script = sb.toString(); - Pair> files = dataHandler.createTransformationRunInfo(context, run, scriptDir, runProperties, batchProperties); - FileLike runInfo = files.getKey(); - - bindings.put(ExternalScriptEngine.WORKING_DIRECTORY, scriptDir.toNioPathForWrite().toString()); - bindings.put(ExternalScriptEngine.SCRIPT_PATH, scriptFile.getAbsolutePath()); - - Map paramMap = new HashMap<>(); - - // Issue 51543: Resolve windows path to run properties - paramMap.put(RUN_INFO_REPLACEMENT, runInfo.toNioPathForWrite().toFile().getAbsolutePath().replaceAll("\\\\", "/")); - - addStandardParameters(context.getRequest(), context.getContainer(), scriptFile, session.getApiKey(), paramMap); - - bindings.put(ExternalScriptEngine.PARAM_REPLACEMENT_MAP, paramMap); - - Object output = engine.eval(script); - - FileLike rewrittenScriptFile = null; - if (bindings.get(ExternalScriptEngine.REWRITTEN_SCRIPT_FILE) instanceof File) - { - var rewrittenScriptFileObject = bindings.get(ExternalScriptEngine.REWRITTEN_SCRIPT_FILE); - if (rewrittenScriptFileObject instanceof FileLike fo) - rewrittenScriptFile = fo; - else - rewrittenScriptFile = FileSystemLike.wrapFile((File)rewrittenScriptFileObject); - } - else - { - rewrittenScriptFile = FileSystemLike.wrapFile(scriptFile); - } - - // process any output from the transformation script - result = dataHandler.processTransformationOutput(context, runInfo, run, rewrittenScriptFile, result, files.getValue()); - - // Propagate any transformed batch properties on to the next script - if (result.getBatchProperties() != null && !result.getBatchProperties().isEmpty()) - { - batchProperties = result.getBatchProperties(); - } - // Propagate any transformed run properties on to the next script - if (result.getRunProperties() != null && !result.getRunProperties().isEmpty()) - { - runProperties = result.getRunProperties(); - } - - context.setTransformResult(result); - } - catch (ValidationException e) - { - throw e; - } - catch (Exception e) - { - ValidationException ve = new ValidationException(e.getMessage() == null ? e.toString() : e.getMessage()); - ve.initCause(e); - throw ve; - } - finally - { - // clean up temp directory - if (!isDefault) - { - try - { - if (null != scriptDir) - { - FileUtil.deleteDir(scriptDir.toNioPathForWrite().toFile()); - FileLike parent = scriptDir.getParent(); - if (parent != null) - parent.delete(); - } - } - catch (IOException e) - { - throw UnexpectedException.wrap(e); - } - } - } - } - else - { - // we may just want to log an error rather than fail the upload due to an engine config problem. - throw new ValidationException("A script engine implementation was not found for the specified QC script (" + scriptFile.getName() + "). " + - "Check configurations in the Admin Console."); - } - } - else - { - throw new ValidationException("The transform script, " + scriptFile.getAbsolutePath() + ", configured for this assay does not exist. Please check " + - "the configuration for this assay design."); - } - } - return result; - } - - public static void addStandardParameters(@Nullable HttpServletRequest request, @Nullable Container container, @Nullable File scriptFile, @Nullable String apiKey, @NotNull Map paramMap) - { - if (scriptFile != null) - { - File srcDir = scriptFile.getParentFile(); - if (srcDir != null && srcDir.exists()) - paramMap.put(SRC_DIR_REPLACEMENT, srcDir.getAbsolutePath().replaceAll("\\\\", "/")); - } - paramMap.put(R_SESSIONID_REPLACEMENT, getSessionInfo(request, apiKey)); - paramMap.put(LEGACY_SESSION_COOKIE_NAME_REPLACEMENT, getSessionCookieName(request)); - paramMap.put(LEGACY_SESSION_ID_REPLACEMENT, getSessionId(request, apiKey)); - paramMap.put(SecurityManager.API_KEY, apiKey); - paramMap.put(BASE_SERVER_URL_REPLACEMENT, AppProps.getInstance().getBaseServerUrl() - + AppProps.getInstance().getContextPath()); - paramMap.put(CONTAINER_PATH, container == null ? null : container.getPath()); - } - - @Deprecated - private static String getSessionCookieName(@Nullable HttpServletRequest request) - { - if (request != null) - { - return CSRFUtil.SESSION_COOKIE_NAME; - } - // issue 19748: need alternative to JSESSIONID for pipeline job transform script usage - return SecurityManager.TRANSFORM_SESSION_ID; - } - - protected FileLike getScriptDir(ExpProtocol protocol, File scriptFile, boolean isDefault) throws IOException - { - FileLike tempDir = FileUtil.getTempDirectoryFileLike(); - FileLike tempRoot = tempDir.resolveChild(ExternalScriptEngine.DEFAULT_WORKING_DIRECTORY); - - if (isDefault && scriptFile.exists()) - { - // TODO getScriptDir(FileLike scriptFile); - tempDir = new FileSystemLike.Builder(scriptFile.getParentFile()).readwrite().root(); - tempRoot = tempDir.resolveChild("TransformAndValidationFiles"); - } - - if (!tempRoot.exists()) - FileUtil.mkdir(tempRoot); - - FileLike tempParent = tempRoot.resolveChild("AssayId_" + protocol.getRowId()); - FileLike tempFolder = AssayFileWriter.findUniqueFileName("work", tempParent); - if (!tempFolder.exists()) - FileUtil.mkdirs(tempFolder); - - return tempFolder; - } - - protected boolean isDefault(ExpProtocol protocol) - { - AssayProvider provider = AssayService.get().getProvider(protocol); - if (provider != null) - return provider.isSaveScriptFiles(protocol); - return false; - } - - /** - * Creates the session information string - */ - private static String getSessionInfo(@Nullable HttpServletRequest request, String apiKey) - { - return "labkey.sessionCookieName = \"" + getSessionCookieName(request) + "\"\n" + - "labkey.sessionCookieContents = \"" + getSessionId(request, apiKey) + "\"\n"; - } - - @Deprecated - private static String getSessionId(@Nullable HttpServletRequest request, String apiKey) - { - if (request != null) - { - return PageFlowUtil.getCookieValue(request.getCookies(), CSRFUtil.SESSION_COOKIE_NAME, ""); - } - return apiKey; - } -} +package org.labkey.api.assay.transform; + +import jakarta.servlet.http.HttpServletRequest; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.assay.AssayFileWriter; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayRunUploadContext; +import org.labkey.api.assay.AssayService; +import org.labkey.api.data.Container; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reader.Readers; +import org.labkey.api.reports.ExternalScriptEngine; +import org.labkey.api.reports.LabKeyScriptEngineManager; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.CSRFUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.UnexpectedException; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import javax.script.Bindings; +import javax.script.ScriptContext; +import javax.script.ScriptEngine; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class DataTransformService +{ + private static final DataTransformService _instance = new DataTransformService(); + + private DataTransformService() {} + + public static DataTransformService get() + { + return _instance; + } + + public static final String RUN_INFO_REPLACEMENT = "runInfo"; + public static final String SRC_DIR_REPLACEMENT = "srcDirectory"; + public static final String R_SESSIONID_REPLACEMENT = "rLabkeySessionId"; + public static final String LEGACY_SESSION_ID_REPLACEMENT = "httpSessionId"; + public static final String LEGACY_SESSION_COOKIE_NAME_REPLACEMENT = "sessionCookieName"; + public static final String BASE_SERVER_URL_REPLACEMENT = "baseServerURL"; + public static final String CONTAINER_PATH = "containerPath"; + public static final String ORIGINAL_SOURCE_PATH = "OriginalSourcePath"; + + public enum TransformOperation + { + INSERT, + UPDATE + } + + public TransformResult transformAndValidate( + AssayRunUploadContext context, + @Nullable ExpRun run, + TransformOperation operation + ) throws ValidationException + { + boolean isDefault = isDefault(context.getProtocol()); + TransformResult result = DefaultTransformResult.createEmptyResult(); + DataExchangeHandler dataHandler = context.getProvider().createDataExchangeHandler(); + if (dataHandler == null) + { + return result; + } + + Map batchProperties; + Map runProperties; + try + { + batchProperties = context.getBatchProperties(); + runProperties = context.getRunProperties(); + } + catch (ExperimentException e) + { + throw new ValidationException(e.getMessage() == null ? e.toString() : e.getMessage()); + } + + for (AnalysisScript analysisScript : context.getProvider().getValidationAndAnalysisScripts(context.getProtocol(), AssayProvider.Scope.ALL)) + { + if (!analysisScript.canExecute(operation)) + continue; + + // read the contents of the script file + File scriptFile = analysisScript.getScript().toNioPathForRead().toFile(); + if (scriptFile.exists()) + { + // TODO: don't bother slurping the contents of binary scripts + StringBuilder sb = new StringBuilder(); + try (BufferedReader br = Readers.getReader(scriptFile)) + { + String l; + while ((l = br.readLine()) != null) + sb.append(l).append('\n'); + } + catch (Exception e) + { + throw new ValidationException(e.getMessage()); + } + ScriptEngine engine = null; + String ext = FileUtil.getExtension(scriptFile); + if (ext != null) + { + engine = LabKeyScriptEngineManager.get() + .getEngineByExtension(context.getContainer(), ext, LabKeyScriptEngineManager.EngineContext.pipeline); + } + if (engine != null) + { + // issue : 46838 remote scripting engines don't support transform scripts yet + if (engine instanceof ExternalScriptEngine externalScriptEngine) + { + if (!externalScriptEngine.supportsContext(LabKeyScriptEngineManager.EngineContext.pipeline)) + throw new ValidationException("The script engine : " + externalScriptEngine.getEngineDefinition().getName() + " does not support running in a transform script." ); + } + FileLike scriptDir = null; + // issue 19748: need alternative to JSESSIONID for pipeline job transform script usage (i.e., TransformSession) + try (SecurityManager.TransformSession session = SecurityManager.createTransformSession(context)) + { + scriptDir = getScriptDir(context.getProtocol(), scriptFile, isDefault); + // issue 13643: ensure script dir is initially empty + FileUtil.deleteDirectoryContents(scriptDir); + + Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); + String script = sb.toString(); + Pair> files = dataHandler.createTransformationRunInfo(context, run, scriptDir, runProperties, batchProperties); + FileLike runInfo = files.getKey(); + + bindings.put(ExternalScriptEngine.WORKING_DIRECTORY, scriptDir.toNioPathForWrite().toString()); + bindings.put(ExternalScriptEngine.SCRIPT_PATH, scriptFile.getAbsolutePath()); + + Map paramMap = new HashMap<>(); + + // Issue 51543: Resolve windows path to run properties + paramMap.put(RUN_INFO_REPLACEMENT, runInfo.toNioPathForWrite().toFile().getAbsolutePath().replaceAll("\\\\", "/")); + + addStandardParameters(context.getRequest(), context.getContainer(), scriptFile, session.getApiKey(), paramMap); + + bindings.put(ExternalScriptEngine.PARAM_REPLACEMENT_MAP, paramMap); + + Object output = engine.eval(script); + + FileLike rewrittenScriptFile = null; + if (bindings.get(ExternalScriptEngine.REWRITTEN_SCRIPT_FILE) instanceof File) + { + var rewrittenScriptFileObject = bindings.get(ExternalScriptEngine.REWRITTEN_SCRIPT_FILE); + if (rewrittenScriptFileObject instanceof FileLike fo) + rewrittenScriptFile = fo; + else + rewrittenScriptFile = FileSystemLike.wrapFile((File)rewrittenScriptFileObject); + } + else + { + rewrittenScriptFile = FileSystemLike.wrapFile(scriptFile); + } + + // process any output from the transformation script + result = dataHandler.processTransformationOutput(context, runInfo, run, rewrittenScriptFile, result, files.getValue()); + + // Propagate any transformed batch properties on to the next script + if (result.getBatchProperties() != null && !result.getBatchProperties().isEmpty()) + { + batchProperties = result.getBatchProperties(); + } + // Propagate any transformed run properties on to the next script + if (result.getRunProperties() != null && !result.getRunProperties().isEmpty()) + { + runProperties = result.getRunProperties(); + } + + context.setTransformResult(result); + } + catch (ValidationException e) + { + throw e; + } + catch (Exception e) + { + ValidationException ve = new ValidationException(e.getMessage() == null ? e.toString() : e.getMessage()); + ve.initCause(e); + throw ve; + } + finally + { + // clean up temp directory + if (!isDefault) + { + try + { + if (null != scriptDir) + { + FileUtil.deleteDir(scriptDir.toNioPathForWrite().toFile()); + FileLike parent = scriptDir.getParent(); + if (parent != null) + parent.delete(); + } + } + catch (IOException e) + { + throw UnexpectedException.wrap(e); + } + } + } + } + else + { + // we may just want to log an error rather than fail the upload due to an engine config problem. + throw new ValidationException("A script engine implementation was not found for the specified QC script (" + scriptFile.getName() + "). " + + "Check configurations in the Admin Console."); + } + } + else + { + throw new ValidationException("The transform script, " + scriptFile.getAbsolutePath() + ", configured for this assay does not exist. Please check " + + "the configuration for this assay design."); + } + } + return result; + } + + public void addStandardParameters(@Nullable HttpServletRequest request, @Nullable Container container, @Nullable File scriptFile, @Nullable String apiKey, @NotNull Map paramMap) + { + if (scriptFile != null) + { + File srcDir = scriptFile.getParentFile(); + if (srcDir != null && srcDir.exists()) + paramMap.put(SRC_DIR_REPLACEMENT, srcDir.getAbsolutePath().replaceAll("\\\\", "/")); + } + paramMap.put(R_SESSIONID_REPLACEMENT, getSessionInfo(request, apiKey)); + paramMap.put(LEGACY_SESSION_COOKIE_NAME_REPLACEMENT, getSessionCookieName(request)); + paramMap.put(LEGACY_SESSION_ID_REPLACEMENT, getSessionId(request, apiKey)); + paramMap.put(SecurityManager.API_KEY, apiKey); + paramMap.put(BASE_SERVER_URL_REPLACEMENT, AppProps.getInstance().getBaseServerUrl() + + AppProps.getInstance().getContextPath()); + paramMap.put(CONTAINER_PATH, container == null ? null : container.getPath()); + } + + @Deprecated + private String getSessionCookieName(@Nullable HttpServletRequest request) + { + if (request != null) + { + return CSRFUtil.SESSION_COOKIE_NAME; + } + // issue 19748: need alternative to JSESSIONID for pipeline job transform script usage + return SecurityManager.TRANSFORM_SESSION_ID; + } + + private FileLike getScriptDir(ExpProtocol protocol, File scriptFile, boolean isDefault) throws IOException + { + FileLike tempDir = FileUtil.getTempDirectoryFileLike(); + FileLike tempRoot = tempDir.resolveChild(ExternalScriptEngine.DEFAULT_WORKING_DIRECTORY); + + if (isDefault && scriptFile.exists()) + { + // TODO getScriptDir(FileLike scriptFile); + tempDir = new FileSystemLike.Builder(scriptFile.getParentFile()).readwrite().root(); + tempRoot = tempDir.resolveChild("TransformAndValidationFiles"); + } + + if (!tempRoot.exists()) + FileUtil.mkdir(tempRoot); + + FileLike tempParent = tempRoot.resolveChild("AssayId_" + protocol.getRowId()); + FileLike tempFolder = AssayFileWriter.findUniqueFileName("work", tempParent); + if (!tempFolder.exists()) + FileUtil.mkdirs(tempFolder); + + return tempFolder; + } + + private boolean isDefault(ExpProtocol protocol) + { + AssayProvider provider = AssayService.get().getProvider(protocol); + if (provider != null) + return provider.isSaveScriptFiles(protocol); + return false; + } + + /** + * Creates the session information string + */ + private String getSessionInfo(@Nullable HttpServletRequest request, String apiKey) + { + return "labkey.sessionCookieName = \"" + getSessionCookieName(request) + "\"\n" + + "labkey.sessionCookieContents = \"" + getSessionId(request, apiKey) + "\"\n"; + } + + @Deprecated + private String getSessionId(@Nullable HttpServletRequest request, String apiKey) + { + if (request != null) + { + return PageFlowUtil.getCookieValue(request.getCookies(), CSRFUtil.SESSION_COOKIE_NAME, ""); + } + return apiKey; + } +} diff --git a/api/src/org/labkey/api/qc/DefaultTransformResult.java b/api/src/org/labkey/api/assay/transform/DefaultTransformResult.java similarity index 88% rename from api/src/org/labkey/api/qc/DefaultTransformResult.java rename to api/src/org/labkey/api/assay/transform/DefaultTransformResult.java index fc9e2cf0b29..79eed13484f 100644 --- a/api/src/org/labkey/api/qc/DefaultTransformResult.java +++ b/api/src/org/labkey/api/assay/transform/DefaultTransformResult.java @@ -1,168 +1,154 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.qc; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.util.HtmlString; -import org.labkey.vfs.FileLike; - -import java.io.File; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * User: klum - * Date: May 7, 2009 - */ -public class DefaultTransformResult implements TransformResult -{ - private Map _dataMap = Collections.emptyMap(); - private Map _batchProperties = Collections.emptyMap(); - private Map _runProperties = Collections.emptyMap(); - private List _uploadedFiles; - private String _assayId; - private String _comments; - private HtmlString _warnings; - private List _files; - private static final Logger LOG = LogManager.getLogger(DefaultTransformResult.class); - - @Override - public List getFiles() - { - return _files; - } - - @Override - public void setFiles(List files) - { - _files = files; - } - - @Override - public HtmlString getWarnings() - { - return _warnings; - } - - @Override - public void setWarnings(HtmlString warnings) - { - _warnings = warnings; - } - - public DefaultTransformResult(){} - - public DefaultTransformResult(Map dataMap) - { - _dataMap = dataMap; - _assayId = null; - } - - public DefaultTransformResult(TransformResult mergeResult) - { - _dataMap = mergeResult.getTransformedData(); - _batchProperties = mergeResult.getBatchProperties(); - _runProperties = mergeResult.getRunProperties(); - _uploadedFiles = mergeResult.getUploadedFiles(); - _warnings = mergeResult.getWarnings(); - _files = mergeResult.getFiles(); - _assayId = null; - } - - @Override - public Map getTransformedData() - { - return _dataMap; - } - - @Override - public Map getBatchProperties() - { - return _batchProperties; - } - - public void setBatchProperties(Map batchProperties) - { - _batchProperties = batchProperties; - } - - @Override - public Map getRunProperties() - { - return _runProperties; - } - - public void setRunProperties(Map runProperties) - { - _runProperties = runProperties; - } - - @Override - public List getUploadedFiles() - { - return _uploadedFiles; - } - - public void setUploadedFiles(List uploadedFiles) - { - _uploadedFiles = uploadedFiles; - } - - public static TransformResult createEmptyResult() - { - return new DefaultTransformResult(); -/* - return new TransformResult() - { - public Map getTransformedData() - { - return Collections.emptyMap(); - } - public boolean isEmpty() {return true;} - }; -*/ - } - - @Override - public String getAssayId() - { - return _assayId; - } - - @Override - public void setAssayId(String assayId) - { - _assayId = assayId; - } - - @Override - public String getComments() - { - return _comments; - } - - @Override - public void setComments(String comments) - { - _comments = comments; - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.assay.transform; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.util.HtmlString; +import org.labkey.vfs.FileLike; + +import java.io.File; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * User: klum + * Date: May 7, 2009 + */ +public class DefaultTransformResult implements TransformResult +{ + private Map _dataMap = Collections.emptyMap(); + private Map _batchProperties = Collections.emptyMap(); + private Map _runProperties = Collections.emptyMap(); + private List _uploadedFiles; + private String _assayId; + private String _comments; + private HtmlString _warnings; + private List _files; + private static final Logger LOG = LogManager.getLogger(DefaultTransformResult.class); + + @Override + public List getFiles() + { + return _files; + } + + public void setFiles(List files) + { + _files = files; + } + + @Override + public HtmlString getWarnings() + { + return _warnings; + } + + public void setWarnings(HtmlString warnings) + { + _warnings = warnings; + } + + public DefaultTransformResult(){} + + public DefaultTransformResult(Map dataMap) + { + _dataMap = dataMap; + _assayId = null; + } + + public DefaultTransformResult(TransformResult mergeResult) + { + _dataMap = mergeResult.getTransformedData(); + _batchProperties = mergeResult.getBatchProperties(); + _runProperties = mergeResult.getRunProperties(); + _uploadedFiles = mergeResult.getUploadedFiles(); + _warnings = mergeResult.getWarnings(); + _files = mergeResult.getFiles(); + _assayId = null; + } + + @Override + public Map getTransformedData() + { + return _dataMap; + } + + @Override + public Map getBatchProperties() + { + return _batchProperties; + } + + public void setBatchProperties(Map batchProperties) + { + _batchProperties = batchProperties; + } + + @Override + public Map getRunProperties() + { + return _runProperties; + } + + public void setRunProperties(Map runProperties) + { + _runProperties = runProperties; + } + + @Override + public List getUploadedFiles() + { + return _uploadedFiles; + } + + public void setUploadedFiles(List uploadedFiles) + { + _uploadedFiles = uploadedFiles; + } + + public static TransformResult createEmptyResult() + { + return new DefaultTransformResult(); + } + + @Override + public String getAssayId() + { + return _assayId; + } + + public void setAssayId(String assayId) + { + _assayId = assayId; + } + + @Override + public String getComments() + { + return _comments; + } + + public void setComments(String comments) + { + _comments = comments; + } +} diff --git a/api/src/org/labkey/api/qc/TransformDataHandler.java b/api/src/org/labkey/api/assay/transform/TransformDataHandler.java similarity index 91% rename from api/src/org/labkey/api/qc/TransformDataHandler.java rename to api/src/org/labkey/api/assay/transform/TransformDataHandler.java index 6aaba938d0d..31b018a8cdc 100644 --- a/api/src/org/labkey/api/qc/TransformDataHandler.java +++ b/api/src/org/labkey/api/assay/transform/TransformDataHandler.java @@ -1,35 +1,32 @@ -/* - * Copyright (c) 2009-2013 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.qc; - -import org.labkey.api.assay.AssayRunUploadContext; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpRun; - -/** - * User: klum - * Date: May 5, 2009 - */ -public interface TransformDataHandler extends ValidationDataHandler -{ - /** - * Imports the data map which may have been transformed by an external script. - */ - void importTransformDataMap(ExpData data, AssayRunUploadContext context, ExpRun run, DataIteratorBuilder dataMap) throws ExperimentException; -} +/* + * Copyright (c) 2009-2013 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.assay.transform; + +import org.labkey.api.assay.AssayRunUploadContext; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.qc.ValidationDataHandler; + +public interface TransformDataHandler extends ValidationDataHandler +{ + /** + * Imports the data map which may have been transformed by an external script. + */ + void importTransformDataMap(ExpData data, AssayRunUploadContext context, ExpRun run, DataIteratorBuilder dataMap) throws ExperimentException; +} diff --git a/api/src/org/labkey/api/qc/TransformResult.java b/api/src/org/labkey/api/assay/transform/TransformResult.java similarity index 65% rename from api/src/org/labkey/api/qc/TransformResult.java rename to api/src/org/labkey/api/assay/transform/TransformResult.java index 23920fcb337..92357a770ff 100644 --- a/api/src/org/labkey/api/qc/TransformResult.java +++ b/api/src/org/labkey/api/assay/transform/TransformResult.java @@ -1,87 +1,64 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.qc; - -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.util.HtmlString; -import org.labkey.vfs.FileLike; - -import java.io.File; -import java.util.List; -import java.util.Map; - -/** - * User: klum - * Date: May 7, 2009 - */ -public interface TransformResult -{ - Map getTransformedData(); - - Map getRunProperties(); - - Map getBatchProperties(); - - List getUploadedFiles(); - - String getAssayId(); - - void setAssayId(String assayId); - - String getComments(); - - void setComments(String comments); - - /** - * Get warnings generated by the transform script. Warnings saved if run properties file severity level - * is set to WARN and transform script writes to errors.html in its working directory. Transform script must - * also set maximum severity in transform run properties to WARN. - * - * @return the string containing the error message - */ - HtmlString getWarnings(); - - /** - * Set transform script warnings. If severity level in run properties level is set to WARN, then these warnings - * will be displayed on the import page along with the output files and the option to proceed with the import or - * cancel the import. This string is displayed directly in an HTML div so HTML syntax is required for any - * formatting or presentation configurations. - * - * @param warnings String containing warnings - */ - void setWarnings(HtmlString warnings); - - /** - * Get transform script output files. These files are collected from the transform scripts operating directory and - * include stdout and stderr piped to file. The output data files, indicated by runDataUploadedFile in the run - * properties file, will also be included. - * - * @return List of files gathered - */ - List getFiles(); - - /** - * Set transform script output files. In the case of a warning being triggered by the transform script, these - * files will be displayed as downloads when the warning and the option to proceed or cancel the import is - * displayed. - * - * @param files List of files to be saved for viewing on warning - */ - void setFiles(List files); -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.assay.transform; + +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.util.HtmlString; +import org.labkey.vfs.FileLike; + +import java.io.File; +import java.util.List; +import java.util.Map; + +/** + * User: klum + * Date: May 7, 2009 + */ +public interface TransformResult +{ + Map getTransformedData(); + + Map getRunProperties(); + + Map getBatchProperties(); + + List getUploadedFiles(); + + String getAssayId(); + + String getComments(); + + /** + * Get warnings generated by the transform script. Warnings saved if run properties file severity level + * is set to WARN and transform script writes to errors.html in its working directory. Transform script must + * also set maximum severity in transform run properties to WARN. + * + * @return the string containing the error message + */ + HtmlString getWarnings(); + + /** + * Get transform script output files. These files are collected from the transform scripts operating directory and + * include stdout and stderr piped to file. The output data files, indicated by runDataUploadedFile in the run + * properties file, will also be included. + * + * @return List of files gathered + */ + List getFiles(); +} diff --git a/api/src/org/labkey/api/qc/DataTransformer.java b/api/src/org/labkey/api/qc/DataTransformer.java deleted file mode 100644 index 2632c24590c..00000000000 --- a/api/src/org/labkey/api/qc/DataTransformer.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2009-2013 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.qc; - -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayRunUploadContext; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.query.ValidationException; - -/** - * Takes assay data and runs it through the configured transform scripts. Each script may transform it into a new - * representation, or simply validate that it meets whatever the custom criteria are. - * - * User: klum - * Date: Sep 22, 2009 - */ -public interface DataTransformer -{ - TransformResult transformAndValidate(AssayRunUploadContext context, ExpRun run) throws ValidationException; -} diff --git a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java index 41880687ab4..5a619eae4e6 100644 --- a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java +++ b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java @@ -31,6 +31,9 @@ import org.labkey.api.assay.TsvDataHandler; import org.labkey.api.assay.actions.AssayRunUploadForm; import org.labkey.api.assay.actions.ProtocolIdForm; +import org.labkey.api.assay.transform.DataExchangeHandler; +import org.labkey.api.assay.transform.DefaultTransformResult; +import org.labkey.api.assay.transform.TransformResult; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.TSVWriter; @@ -632,7 +635,6 @@ else if (null != maxSeverity && maxSeverity.equals(errLevel.ERROR.name())) throw new ValidationException("Transform script has thrown errors."); } } - } public void processValidationOutput(RunInfo info, @Nullable Logger log) throws ValidationException diff --git a/api/src/org/labkey/api/qc/TsvDataSerializer.java b/api/src/org/labkey/api/qc/TsvDataSerializer.java index 46e753cc378..cf73bcccae7 100644 --- a/api/src/org/labkey/api/qc/TsvDataSerializer.java +++ b/api/src/org/labkey/api/qc/TsvDataSerializer.java @@ -20,6 +20,7 @@ import org.labkey.api.assay.AbstractAssayTsvDataHandler; import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; +import org.labkey.api.assay.transform.DataExchangeHandler; import org.labkey.api.data.TSVWriter; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.dataiterator.DataIteratorContext; diff --git a/api/src/org/labkey/api/reports/report/ExternalScriptEngineReport.java b/api/src/org/labkey/api/reports/report/ExternalScriptEngineReport.java index b1cb5f20be8..7c573cb0f38 100644 --- a/api/src/org/labkey/api/reports/report/ExternalScriptEngineReport.java +++ b/api/src/org/labkey/api/reports/report/ExternalScriptEngineReport.java @@ -19,7 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.labkey.api.ApiModule; -import org.labkey.api.assay.DefaultDataTransformer; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.attachments.AttachmentParent; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.module.ModuleLoader; @@ -345,7 +345,7 @@ protected Object runScript(ScriptEngine engine, ViewContext context, List paramMap = new HashMap<>(); - DefaultDataTransformer.addStandardParameters(null, context.getContainer(), null, session.getApiKey(), paramMap); + DataTransformService.get().addStandardParameters(null, context.getContainer(), null, session.getApiKey(), paramMap); bindings.put(ExternalScriptEngine.PARAM_REPLACEMENT_MAP, paramMap); if (engine instanceof RserveScriptEngine) diff --git a/assay/api-src/org/labkey/api/assay/dilution/AbstractDilutionAssayProvider.java b/assay/api-src/org/labkey/api/assay/dilution/AbstractDilutionAssayProvider.java index 18fc5c1b4c3..885cdee8c1a 100644 --- a/assay/api-src/org/labkey/api/assay/dilution/AbstractDilutionAssayProvider.java +++ b/assay/api-src/org/labkey/api/assay/dilution/AbstractDilutionAssayProvider.java @@ -24,7 +24,7 @@ import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.exp.property.Lookup; import org.labkey.api.module.Module; -import org.labkey.api.qc.DataExchangeHandler; +import org.labkey.api.assay.transform.DataExchangeHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.security.User; import org.labkey.api.assay.plate.AbstractPlateBasedAssayProvider; diff --git a/assay/src/org/labkey/assay/AssayController.java b/assay/src/org/labkey/assay/AssayController.java index a15e940aee8..11f6f921e18 100644 --- a/assay/src/org/labkey/assay/AssayController.java +++ b/assay/src/org/labkey/assay/AssayController.java @@ -94,7 +94,7 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.qc.DataExchangeHandler; +import org.labkey.api.assay.transform.DataExchangeHandler; import org.labkey.api.qc.DataState; import org.labkey.api.qc.DataStateManager; import org.labkey.api.query.BatchValidationException; diff --git a/assay/src/org/labkey/assay/AssayDomainServiceImpl.java b/assay/src/org/labkey/assay/AssayDomainServiceImpl.java index 9616cc8854d..69007fcee1f 100644 --- a/assay/src/org/labkey/assay/AssayDomainServiceImpl.java +++ b/assay/src/org/labkey/assay/AssayDomainServiceImpl.java @@ -32,6 +32,8 @@ import org.labkey.api.assay.plate.PlateBasedAssayProvider; import org.labkey.api.assay.plate.PlateService; import org.labkey.api.assay.security.DesignAssayPermission; +import org.labkey.api.assay.transform.AnalysisScript; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -247,12 +249,12 @@ private GWTProtocol getAssayTemplate(AssayProvider provider, Pair typeScripts = provider.getValidationAndAnalysisScripts(protocol, AssayProvider.Scope.ASSAY_TYPE); + List typeScripts = provider.getValidationAndAnalysisScripts(protocol, AssayProvider.Scope.ASSAY_TYPE); if (!typeScripts.isEmpty()) { List scriptNames = new ArrayList<>(); - for (File script : typeScripts) - scriptNames.add(script.getAbsolutePath()); + for (AnalysisScript script : typeScripts) + scriptNames.add(script.getScript().toNioPathForRead().toString()); result.setModuleTransformScripts(scriptNames); } @@ -264,12 +266,13 @@ private GWTProtocol getAssayTemplate(AssayProvider provider, Pair transformScripts = provider.getValidationAndAnalysisScripts(protocol, AssayProvider.Scope.ASSAY_DEF); + List transformScripts = provider.getValidationAndAnalysisScripts(protocol, AssayProvider.Scope.ASSAY_DEF); List transformScriptStrings = new ArrayList<>(); - for (File transformScript : transformScripts) + for (AnalysisScript transformScript : transformScripts) { - transformScriptStrings.add(transformScript.getAbsolutePath()); + // TODO, add allowable operations once we have UI that controls those options + transformScriptStrings.add(transformScript.getScript().toNioPathForRead().toString()); } result.setProtocolTransformScripts(transformScriptStrings); @@ -490,7 +493,7 @@ public GWTProtocol saveChanges(GWTProtocol assay, boolean replaceIfExisting) thr } // data transform scripts - List transformScripts = new ArrayList<>(); + List transformScripts = new ArrayList<>(); List submittedScripts = assay.getProtocolTransformScripts(); if (!submittedScripts.isEmpty() && !canUpdateTransformationScript()) throw new ValidationException("You must be a platform developer or site admin to configure assay transformation scripts."); @@ -498,7 +501,8 @@ public GWTProtocol saveChanges(GWTProtocol assay, boolean replaceIfExisting) thr { if (!StringUtils.isBlank(script)) { - transformScripts.add(new File(script)); + // TODO : handle analysis script operations once UI is created + transformScripts.add(new AnalysisScript(new File(script), Set.of(DataTransformService.TransformOperation.INSERT))); } } diff --git a/assay/src/org/labkey/assay/AssayModule.java b/assay/src/org/labkey/assay/AssayModule.java index ded8dbba4d6..0fcf3680ee7 100644 --- a/assay/src/org/labkey/assay/AssayModule.java +++ b/assay/src/org/labkey/assay/AssayModule.java @@ -102,8 +102,8 @@ import java.util.List; import java.util.Set; -import static org.labkey.api.assay.DefaultDataTransformer.LEGACY_SESSION_COOKIE_NAME_REPLACEMENT; -import static org.labkey.api.assay.DefaultDataTransformer.LEGACY_SESSION_ID_REPLACEMENT; +import static org.labkey.api.assay.transform.DataTransformService.LEGACY_SESSION_COOKIE_NAME_REPLACEMENT; +import static org.labkey.api.assay.transform.DataTransformService.LEGACY_SESSION_ID_REPLACEMENT; public class AssayModule extends SpringModule { diff --git a/assay/src/org/labkey/assay/ModuleAssayProvider.java b/assay/src/org/labkey/assay/ModuleAssayProvider.java index 262e0d368cb..7c77375b62b 100644 --- a/assay/src/org/labkey/assay/ModuleAssayProvider.java +++ b/assay/src/org/labkey/assay/ModuleAssayProvider.java @@ -34,6 +34,8 @@ import org.labkey.api.assay.TsvDataHandler; import org.labkey.api.assay.actions.AssayRunUploadForm; import org.labkey.api.assay.actions.UploadWizardAction; +import org.labkey.api.assay.transform.AnalysisScript; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.data.Container; import org.labkey.api.exp.Lsid; import org.labkey.api.exp.XarContext; @@ -73,7 +75,6 @@ import org.springframework.validation.BindException; import org.springframework.web.servlet.ModelAndView; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -663,10 +664,10 @@ public boolean hasUsefulDetailsPage() @NotNull @Override - public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope) + public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope) { // Start with the standard set - List result = new ArrayList<>(super.getValidationAndAnalysisScripts(protocol, scope)); + List result = new ArrayList<>(super.getValidationAndAnalysisScripts(protocol, scope)); if (scope == Scope.ASSAY_TYPE || scope == Scope.ALL) { @@ -678,7 +679,7 @@ public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope sc final LabKeyScriptEngineManager manager = LabKeyScriptEngineManager.get(); Collection scripts = scriptDir.list(); - List moduleScriptFiles = new ArrayList<>(scripts.size()); + List moduleScriptFiles = new ArrayList<>(scripts.size()); for (Resource r : scripts) { if (r instanceof FileResource) @@ -687,7 +688,8 @@ public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope sc FileResource fileResource = (FileResource) r; if (manager.getEngineByExtension(protocol.getContainer(), ext, LabKeyScriptEngineManager.EngineContext.pipeline) != null) { - moduleScriptFiles.add(fileResource.getFile()); + // only allow insert execution for module scripts + moduleScriptFiles.add(new AnalysisScript(fileResource.getFile(), Set.of(DataTransformService.TransformOperation.INSERT))); } else { @@ -702,10 +704,10 @@ public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope sc } // Put the scripts in the order specified by the config.xml file - List sortedModuleScripts = new ArrayList<>(); + List sortedModuleScripts = new ArrayList<>(); for (ScriptMetadata scriptMetadata : _scriptMetadata) { - File matchingScript = findAndRemove(moduleScriptFiles, scriptMetadata.getFileName()); + AnalysisScript matchingScript = findAndRemove(moduleScriptFiles, scriptMetadata.getFileName()); if (matchingScript != null) { sortedModuleScripts.add(matchingScript); @@ -714,7 +716,7 @@ public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope sc result.addAll(sortedModuleScripts); // Add any remaining module-provided files in alphabetical order - moduleScriptFiles.sort(Comparator.comparing(File::getName, String.CASE_INSENSITIVE_ORDER)); + moduleScriptFiles.sort(Comparator.comparing(s -> s.getScript().getName(), String.CASE_INSENSITIVE_ORDER)); result.addAll(moduleScriptFiles); } } @@ -722,14 +724,14 @@ public List getValidationAndAnalysisScripts(ExpProtocol protocol, Scope sc } /** Finds a script from the list of available files, and removes it from the list */ - private File findAndRemove(List scriptFiles, String fileName) + private AnalysisScript findAndRemove(List scripts, String fileName) { - for (File scriptFile : scriptFiles) + for (AnalysisScript script : scripts) { - if (scriptFile.getName().equalsIgnoreCase(fileName)) + if (script.getScript().getName().equalsIgnoreCase(fileName)) { - scriptFiles.remove(scriptFile); - return scriptFile; + scripts.remove(script); + return script; } } // Only warn the first time we notice that there's a script that's in the config.xml file but not on disk diff --git a/assay/src/org/labkey/assay/TsvAssayProvider.java b/assay/src/org/labkey/assay/TsvAssayProvider.java index a466b04fbfa..9036fade7c7 100644 --- a/assay/src/org/labkey/assay/TsvAssayProvider.java +++ b/assay/src/org/labkey/assay/TsvAssayProvider.java @@ -68,7 +68,7 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineProvider; import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.qc.DataExchangeHandler; +import org.labkey.api.assay.transform.DataExchangeHandler; import org.labkey.api.qc.TsvDataExchangeHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.security.User; diff --git a/pipeline/src/org/labkey/pipeline/analysis/CommandTaskImpl.java b/pipeline/src/org/labkey/pipeline/analysis/CommandTaskImpl.java index 1ec5d0e4ef3..af052b9991d 100644 --- a/pipeline/src/org/labkey/pipeline/analysis/CommandTaskImpl.java +++ b/pipeline/src/org/labkey/pipeline/analysis/CommandTaskImpl.java @@ -22,7 +22,7 @@ import org.jmock.lib.legacy.ClassImposteriser; import org.junit.Assert; import org.junit.Test; -import org.labkey.api.assay.DefaultDataTransformer; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.data.Container; import org.labkey.api.exp.PropertyType; import org.labkey.api.module.Module; @@ -550,7 +550,7 @@ else if (inputPaths.length == 1) String[] originalFiles = getOriginalFiles(key); if (originalFiles.length == 1) { - replacements.put(DefaultDataTransformer.ORIGINAL_SOURCE_PATH, Matcher.quoteReplacement(originalFiles[0].replaceAll("\\\\", "/"))); + replacements.put(DataTransformService.ORIGINAL_SOURCE_PATH, Matcher.quoteReplacement(originalFiles[0].replaceAll("\\\\", "/"))); } } @@ -601,7 +601,7 @@ else if (outputPaths.length == 1) replacements.put(PipelineJob.PIPELINE_TASK_OUTPUT_PARAMS_PARAM, taskOutputParamsRelativePath); } - DefaultDataTransformer.addStandardParameters(null, container, scriptFile, apiKey, replacements); + DataTransformService.get().addStandardParameters(null, container, scriptFile, apiKey, replacements); return replacements; } diff --git a/pipeline/src/org/labkey/pipeline/api/SimpleTaskFactory.java b/pipeline/src/org/labkey/pipeline/api/SimpleTaskFactory.java index b5c33574f39..223c8337c36 100644 --- a/pipeline/src/org/labkey/pipeline/api/SimpleTaskFactory.java +++ b/pipeline/src/org/labkey/pipeline/api/SimpleTaskFactory.java @@ -20,7 +20,7 @@ import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Test; -import org.labkey.api.assay.DefaultDataTransformer; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; @@ -79,9 +79,9 @@ public abstract class SimpleTaskFactory extends CommandTaskImpl.Factory PipelineJob.PIPELINE_TASK_INFO_PARAM, PipelineJob.PIPELINE_TASK_OUTPUT_PARAMS_PARAM, // The following replacements aren't used yet, but are reserved for future use. - DefaultDataTransformer.RUN_INFO_REPLACEMENT, - DefaultDataTransformer.SRC_DIR_REPLACEMENT, - DefaultDataTransformer.R_SESSIONID_REPLACEMENT + DataTransformService.RUN_INFO_REPLACEMENT, + DataTransformService.SRC_DIR_REPLACEMENT, + DataTransformService.R_SESSIONID_REPLACEMENT ); protected Map _params; diff --git a/study/src/org/labkey/study/pipeline/FileAnalysisDatasetTask.java b/study/src/org/labkey/study/pipeline/FileAnalysisDatasetTask.java index 73b1ad37fbd..7a37571dda9 100644 --- a/study/src/org/labkey/study/pipeline/FileAnalysisDatasetTask.java +++ b/study/src/org/labkey/study/pipeline/FileAnalysisDatasetTask.java @@ -19,7 +19,7 @@ import org.labkey.api.action.BaseViewAction; import org.labkey.api.action.NullSafeBindException; import org.labkey.api.admin.PipelineJobLoggerGetter; -import org.labkey.api.assay.DefaultDataTransformer; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedActionSet; @@ -102,8 +102,8 @@ public RecordedActionSet run() throws PipelineJobException inputDataMap.put(file, new Pair<>(DATASET_ID_KEY, params.get(DATASET_ID_KEY))); else if (params.containsKey(DATASET_NAME_KEY)) inputDataMap.put(file, new Pair<>(DATASET_NAME_KEY, params.get(DATASET_NAME_KEY))); - else if (params.containsKey(DefaultDataTransformer.ORIGINAL_SOURCE_PATH)) - inputDataMap.put(file, new Pair<>(DefaultDataTransformer.ORIGINAL_SOURCE_PATH, params.get(DefaultDataTransformer.ORIGINAL_SOURCE_PATH))); + else if (params.containsKey(DataTransformService.ORIGINAL_SOURCE_PATH)) + inputDataMap.put(file, new Pair<>(DataTransformService.ORIGINAL_SOURCE_PATH, params.get(DataTransformService.ORIGINAL_SOURCE_PATH))); } List readerErrors = new ArrayList<>(); StudyImpl study = getStudy(); From 15e395abe40d20e7b7b84bf29ba91b8b0cbb225c Mon Sep 17 00:00:00 2001 From: Lum Date: Thu, 19 Dec 2024 12:31:03 -0800 Subject: [PATCH 2/2] code review feedback --- .../labkey/api/assay/transform/AnalysisScript.java | 14 +++++++++++++- .../org/labkey/assay/AssayDomainServiceImpl.java | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/api/src/org/labkey/api/assay/transform/AnalysisScript.java b/api/src/org/labkey/api/assay/transform/AnalysisScript.java index a4f3134606c..c10574f7e38 100644 --- a/api/src/org/labkey/api/assay/transform/AnalysisScript.java +++ b/api/src/org/labkey/api/assay/transform/AnalysisScript.java @@ -30,7 +30,14 @@ private AnalysisScript(File script, List operations) { _script = new FileSystemLike.Builder(script).build().getRoot(); for (String op : operations) - _operations.add(DataTransformService.TransformOperation.valueOf(op)); + { + if (op != null) + _operations.add(DataTransformService.TransformOperation.valueOf(op)); + } + + // default to insert only + if (_operations.isEmpty()) + _operations.add(DataTransformService.TransformOperation.INSERT); } public FileLike getScript() @@ -38,6 +45,11 @@ public FileLike getScript() return _script; } + public String getScriptPath() + { + return _script.toNioPathForRead().toString(); + } + public boolean canExecute(DataTransformService.TransformOperation operation) { return _operations.contains(operation); diff --git a/assay/src/org/labkey/assay/AssayDomainServiceImpl.java b/assay/src/org/labkey/assay/AssayDomainServiceImpl.java index cc093864c9b..e172933736c 100644 --- a/assay/src/org/labkey/assay/AssayDomainServiceImpl.java +++ b/assay/src/org/labkey/assay/AssayDomainServiceImpl.java @@ -254,7 +254,7 @@ private GWTProtocol getAssayTemplate(AssayProvider provider, Pair scriptNames = new ArrayList<>(); for (AnalysisScript script : typeScripts) - scriptNames.add(script.getScript().toNioPathForRead().toString()); + scriptNames.add(script.getScriptPath()); result.setModuleTransformScripts(scriptNames); } @@ -272,7 +272,7 @@ private GWTProtocol getAssayTemplate(AssayProvider provider, Pair