From 546d6f9f1bda252a83aac48cf6e4bc28326eb45f Mon Sep 17 00:00:00 2001 From: ian Date: Fri, 21 Mar 2025 16:25:39 -0700 Subject: [PATCH 1/9] Convert File usages to use FileLike or the comparable class --- .../genotyping/GenotypingController.java | 27 +++++---- .../labkey/genotyping/GenotypingManager.java | 29 ++++------ .../org/labkey/genotyping/GenotypingRun.java | 17 +++--- .../labkey/genotyping/ImportAnalysisJob.java | 16 +++--- .../src/org/labkey/genotyping/ReadsJob.java | 4 +- .../labkey/genotyping/SubmitAnalysisJob.java | 44 +++++++++------ .../galaxy/WorkflowCompletionMonitor.java | 56 +++++++++---------- 7 files changed, 97 insertions(+), 96 deletions(-) diff --git a/genotyping/src/org/labkey/genotyping/GenotypingController.java b/genotyping/src/org/labkey/genotyping/GenotypingController.java index c4bf8c77..e468f82b 100644 --- a/genotyping/src/org/labkey/genotyping/GenotypingController.java +++ b/genotyping/src/org/labkey/genotyping/GenotypingController.java @@ -18,7 +18,6 @@ import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -86,10 +85,12 @@ import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.util.DateUtil; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.MinorConfigurationException; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Pair; import org.labkey.api.util.URLHelper; +import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.DataView; import org.labkey.api.view.DetailsView; @@ -110,6 +111,8 @@ import org.labkey.genotyping.sequences.FastqGenerator; import org.labkey.genotyping.sequences.FastqWriter; import org.labkey.genotyping.sequences.SequenceManager; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -122,6 +125,7 @@ import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Paths; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; @@ -139,7 +143,7 @@ public class GenotypingController extends SpringActionController { - private static final Logger LOG = LogManager.getLogger(GenotypingController.class); + private static final Logger LOG = LogHelper.getLogger(GenotypingController.class, "Logger for Genotyping Controller"); @SuppressWarnings({"unchecked"}) private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(GenotypingController.class); @@ -1007,7 +1011,7 @@ private String importReads(ImportReadsForm form) throws Exception try { - File readsFile = new File(form.getReadsPath()); + FileLike readsFile = new FileSystemLike.Builder(FileUtil.createUri(form.getReadsPath())).readonly().root(); GenotypingRun run; try @@ -1178,7 +1182,7 @@ private ColumnInfo getColumnInfo(Map fieldMap, String colu public boolean handlePost(AnalyzeForm form, BindException errors) throws Exception { GenotypingRun run = GenotypingManager.get().getRun(getContainer(), form.getRun()); - File readsPath = new File(run.getPath(), run.getFileName()); + FileLike readsPath = run.getWorkingDir().resolveChild(run.getFileName()); ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), getViewContext().getActionURL()); PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); @@ -1293,7 +1297,8 @@ public Object execute(ImportAnalysisForm form, BindException errors) throws Exce try { int analysisId = form.getAnalysis(); - File analysisDir = new File(form.getPath()); + FileLike analysisDir = new FileSystemLike.Builder(Paths.get(form.getPath())).readwrite().root(); + User user = getUser(); if (user.isGuest()) @@ -1379,8 +1384,8 @@ public void validateCommand(PipelinePathForm target, Errors errors) public boolean handlePost(PipelinePathForm form, BindException errors) throws IOException, PipelineValidationException { // Manual upload of genotyping analysis; pipeline provider posts to this action with matches file. - File matches = form.getValidatedSingleFile(getContainer()); - File analysisDir = matches.getParentFile(); + FileLike matches = FileSystemLike.wrapFile(form.getValidatedSingleFile(getContainer())); + FileLike analysisDir = matches.getParent(); // Load properties to determine the run. Properties props = GenotypingManager.get().readProperties(analysisDir); @@ -1398,13 +1403,13 @@ public URLHelper getSuccessURL(PipelinePathForm pipelinePathForm) } - private void importAnalysis(int analysisId, File pipelineDir, User user) throws IOException, PipelineValidationException + private void importAnalysis(int analysisId, FileLike pipelineDir, User user) throws IOException, PipelineValidationException { GenotypingAnalysis analysis = GenotypingManager.get().getAnalysis(getContainer(), analysisId); - File analysisDir = new File(analysis.getPath()); + FileLike analysisDir = new FileSystemLike.Builder(Paths.get(analysis.getPath())).readwrite().root(); - String pipelinePath = pipelineDir.getCanonicalPath(); - String analysisPath = analysisDir.getCanonicalPath(); + String pipelinePath = FileUtil.getAbsoluteCaseSensitiveFile(pipelineDir.toNioPathForRead().toFile()).getAbsolutePath(); + String analysisPath = FileUtil.getAbsoluteCaseSensitiveFile(analysisDir.toNioPathForRead().toFile()).getAbsolutePath(); if (!pipelinePath.equals(analysisPath)) throw new FileNotFoundException("Analysis path (\"" + analysisPath + diff --git a/genotyping/src/org/labkey/genotyping/GenotypingManager.java b/genotyping/src/org/labkey/genotyping/GenotypingManager.java index ff71aff4..4bba3670 100644 --- a/genotyping/src/org/labkey/genotyping/GenotypingManager.java +++ b/genotyping/src/org/labkey/genotyping/GenotypingManager.java @@ -32,13 +32,12 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryHelper; import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.SafeToRenderEnum; import org.labkey.api.view.NotFoundException; +import org.labkey.vfs.FileLike; -import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -124,7 +123,7 @@ public void saveSettings(Container c, GenotypingFolderSettings settings) map.save(); } - public GenotypingRun createRun(Container c, User user, Integer metaDataId, File readsFile, String platform) + public GenotypingRun createRun(Container c, User user, Integer metaDataId, FileLike readsFile, String platform) { MetaDataRun mdRun = null; @@ -281,31 +280,23 @@ private void deleteAnalyses(CharSequence analysisFilter, Object... params) } - public void writeProperties(Properties props, File directory) throws IOException + public void writeProperties(Properties props, FileLike directory) throws IOException { - File propXml = new File(directory, PROPERTIES_FILE_NAME); - OutputStream os = null; - try + try (OutputStream os = directory.resolveChild(PROPERTIES_FILE_NAME).openOutputStream()) { - os = new FileOutputStream(propXml); props.storeToXML(os, null); } - finally - { - if (null != os) - os.close(); - } } - public Properties readProperties(File directory) throws IOException + public Properties readProperties(FileLike directory) throws IOException { if (!directory.exists()) - throw new FileNotFoundException(directory.getAbsolutePath() + " does not exist"); + throw new FileNotFoundException(FileUtil.getAbsolutePath(directory.toNioPathForRead()) + " does not exist"); if (!directory.isDirectory()) - throw new FileNotFoundException(directory.getAbsolutePath() + " is not a directory"); + throw new FileNotFoundException(FileUtil.getAbsolutePath(directory.toNioPathForRead()) + " is not a directory"); - File properties = new File(directory, PROPERTIES_FILE_NAME); + FileLike properties = directory.resolveChild(PROPERTIES_FILE_NAME); // Load properties to determine the run. Properties props = new Properties(); @@ -313,7 +304,7 @@ public Properties readProperties(File directory) throws IOException try { - is = new FileInputStream(properties); + is = properties.openInputStream(); props.loadFromXML(is); } finally diff --git a/genotyping/src/org/labkey/genotyping/GenotypingRun.java b/genotyping/src/org/labkey/genotyping/GenotypingRun.java index 0736c3ee..8db217cd 100644 --- a/genotyping/src/org/labkey/genotyping/GenotypingRun.java +++ b/genotyping/src/org/labkey/genotyping/GenotypingRun.java @@ -18,10 +18,9 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; import org.labkey.api.util.MemTracker; +import org.labkey.vfs.FileLike; -import java.io.File; import java.util.Date; /** @@ -35,7 +34,7 @@ public class GenotypingRun private Container _container; private int _createdBy; private Date _created; - private String _path; + private FileLike _workingDir; private String _platform; private String _fileName; private Integer _metaDataId = null; @@ -46,11 +45,11 @@ public GenotypingRun() MemTracker.getInstance().put(this); } - public GenotypingRun(Container c, File readsFile, @Nullable MetaDataRun metaDataRun, String platform) + public GenotypingRun(Container c, FileLike readsFile, @Nullable MetaDataRun metaDataRun, String platform) { this(); setContainer(c); - setPath(FileUtil.getAbsoluteCaseSensitiveFile(readsFile.getParentFile()).getPath()); + setWorkingDir(readsFile.getParent()); setFileName(readsFile.getName()); setPlatform(platform); @@ -118,14 +117,14 @@ public void setCreated(Date created) _created = created; } - public String getPath() + public FileLike getWorkingDir() { - return _path; + return _workingDir; } - public void setPath(String path) + public void setWorkingDir(FileLike path) { - _path = path; + _workingDir = path; } public String getPlatform() diff --git a/genotyping/src/org/labkey/genotyping/ImportAnalysisJob.java b/genotyping/src/org/labkey/genotyping/ImportAnalysisJob.java index 51999c13..55d29ada 100644 --- a/genotyping/src/org/labkey/genotyping/ImportAnalysisJob.java +++ b/genotyping/src/org/labkey/genotyping/ImportAnalysisJob.java @@ -36,8 +36,8 @@ import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.genotyping.sequences.SequenceDictionary; import org.labkey.genotyping.sequences.SequenceManager; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.Map; @@ -50,21 +50,21 @@ */ public class ImportAnalysisJob extends PipelineJob { - private File _dir; + private FileLike _dir; private GenotypingAnalysis _analysis; // For serialization protected ImportAnalysisJob() {} - public ImportAnalysisJob(ViewBackgroundInfo info, PipeRoot root, File pipelineDir, GenotypingAnalysis analysis) + public ImportAnalysisJob(ViewBackgroundInfo info, PipeRoot root, FileLike pipelineDir, GenotypingAnalysis analysis) { super("Import Analysis", info, root); _dir = pipelineDir; _analysis = analysis; - setLogFile(new File(_dir, FileUtil.makeFileNameWithTimestamp("import_analysis", "log"))); + setLogFile(_dir.resolveChild(FileUtil.makeFileNameWithTimestamp("import_analysis", "log")).toNioPathForWrite()); if (!_dir.exists()) - throw new IllegalArgumentException("Pipeline directory does not exist: " + _dir.getAbsolutePath()); + throw new IllegalArgumentException("Pipeline directory does not exist: " + _dir.toNioPathForRead().toAbsolutePath()); if (null == _analysis) throw new IllegalArgumentException("Analysis was not specified"); @@ -92,7 +92,7 @@ public void run() try { - File sourceMatches = new File(_dir, GenotypingManager.MATCHES_FILE_NAME); + FileLike sourceMatches = _dir.resolveChild(GenotypingManager.MATCHES_FILE_NAME); GenotypingSchema gs = GenotypingSchema.get(); DbSchema schema = gs.getSchema(); TempTableInfo matches = null; @@ -180,9 +180,9 @@ public void run() // columnNames: comma-separated list of column names to include; null means include all columns - private TempTableInfo createTempTable(File file, @Nullable String columnNames) throws IOException, SQLException + private TempTableInfo createTempTable(FileLike file, @Nullable String columnNames) throws IOException, SQLException { - try (TabLoader loader = new TabLoader(file, true)) + try (TabLoader loader = new TabLoader(file.toNioPathForRead().toFile(), true)) { // Load only the specified columns if (null != columnNames) diff --git a/genotyping/src/org/labkey/genotyping/ReadsJob.java b/genotyping/src/org/labkey/genotyping/ReadsJob.java index 3929a4e5..9ef1d4bd 100644 --- a/genotyping/src/org/labkey/genotyping/ReadsJob.java +++ b/genotyping/src/org/labkey/genotyping/ReadsJob.java @@ -35,8 +35,8 @@ import org.labkey.api.util.MailHelper; import org.labkey.api.util.URLHelper; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; -import java.io.File; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -79,7 +79,7 @@ public void updateRunStatus(Status status) { try { - File file = new File(_run.getPath(), _run.getFileName()); + FileLike file = _run.getWorkingDir().resolveChild(_run.getFileName()); GenotypingRun newRun = GenotypingManager.get().createRun(getContainer(), getUser(), _run.getMetaDataId(), file, _run.getPlatform()); _run.setRowId(newRun.getRowId()); } diff --git a/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java b/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java index f4fb4248..b7e822b1 100644 --- a/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java +++ b/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java @@ -39,9 +39,11 @@ import org.labkey.genotyping.galaxy.GalaxyUtils; import org.labkey.genotyping.galaxy.WorkflowCompletionMonitor; import org.labkey.genotyping.sequences.SequenceManager; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; +import java.io.OutputStream; import java.net.URISyntaxException; import java.sql.SQLException; import java.util.HashMap; @@ -56,13 +58,13 @@ */ public class SubmitAnalysisJob extends PipelineJob { - private final File _dir; + private final FileLike _dir; private final GenotypingAnalysis _analysis; - private final File _analysisDir; + private final FileLike _analysisDir; private final Set _sampleIds; private URLHelper _galaxyURL = null; - private File _completionFile = null; // Used for dev mode only + private FileLike _completionFile = null; // Used for dev mode only // In dev mode only, we'll test the ability to connect to the Galaxy server once; if this connection fails, we'll // skip trying to submit to Galaxy on subsequent attempts (until server restart). @@ -70,12 +72,12 @@ public class SubmitAnalysisJob extends PipelineJob @JsonCreator protected SubmitAnalysisJob( - @JsonProperty("_dir") File dir, + @JsonProperty("_dir") FileLike dir, @JsonProperty("_analysis") GenotypingAnalysis analysis, - @JsonProperty("_analysisDir") File analysisDir, + @JsonProperty("_analysisDir") FileLike analysisDir, @JsonProperty("_sampleIds") Set sampleIds, @JsonProperty("_galaxyURL") URLHelper galaxyURL, - @JsonProperty("_completionFile") File completionFile, + @JsonProperty("_completionFile") FileLike completionFile, @JsonProperty("_useGalaxy") Boolean useGalaxy ) { @@ -88,24 +90,30 @@ protected SubmitAnalysisJob( _useGalaxy = useGalaxy; } - public SubmitAnalysisJob(ViewBackgroundInfo info, PipeRoot root, File reads, GenotypingAnalysis analysis, @NotNull Set sampleIds) + public SubmitAnalysisJob(ViewBackgroundInfo info, PipeRoot root, FileLike reads, GenotypingAnalysis analysis, @NotNull Set sampleIds) { super("Submit Analysis", info, root); // No pipeline provider - _dir = reads.getParentFile(); + _dir = reads.getParent(); _analysis = analysis; _sampleIds = sampleIds; - _analysisDir = new File(_dir, "analysis_" + _analysis.getRowId()); + _analysisDir = _dir.resolveChild("analysis_" + _analysis.getRowId()); if (_analysisDir.exists()) throw new MinorConfigurationException("Analysis directory already exists: " + _analysisDir.getPath()); - if (!_analysisDir.mkdir()) + try + { + _analysisDir.mkdir(); + } + catch(IOException e) + { throw new MinorConfigurationException("Can't create analysis directory: " + _analysisDir.getPath()); + } - setLogFile(new File(_analysisDir, FileUtil.makeFileNameWithTimestamp("submit_analysis", "log")).toPath()); + setLogFile(_analysisDir.resolveChild(FileUtil.makeFileNameWithTimestamp("submit_analysis", "log")).toNioPathForWrite()); info("Creating analysis directory: " + _analysisDir.getName()); - _analysis.setPath(_analysisDir.getAbsolutePath()); + _analysis.setPath(_analysisDir.getPath().toString()); _analysis.setFileName(_analysisDir.getName()); Table.update(getUser(), GenotypingSchema.get().getAnalysesTable(), PageFlowUtil.map("Path", _analysis.getPath(), "FileName", _analysis.getFileName()), _analysis.getRowId()); } @@ -199,9 +207,9 @@ protected int write() return rows.getValue(); } - }) + }; OutputStream os = _analysisDir.resolveChild("reads.txt").openOutputStream()) { - writer.write(new File(_analysisDir, "reads.txt")); + writer.write(os); } } @@ -220,7 +228,7 @@ private void writeProperties(@Nullable GalaxyServer server) throws IOException // the Galaxy server can't communicate via HTTP with LabKey Server, so watch for this file as a backup plan. if (AppProps.getInstance().isDevMode() || null == server) { - _completionFile = new File(_analysisDir, "analysis_complete.txt"); + _completionFile = _analysisDir.resolveChild("analysis_complete.txt"); if (_completionFile.exists()) throw new IllegalStateException("Completion file already exists: " + _completionFile.getPath()); @@ -236,7 +244,7 @@ private void writeFasta() throws IOException, SQLException { info("Writing FASTA file"); setStatus("WRITING FASTA"); - File fastaFile = new File(_analysisDir, GenotypingManager.SEQUENCES_FILE_NAME); + File fastaFile = _analysisDir.resolveChild(GenotypingManager.SEQUENCES_FILE_NAME).toNioPathForWrite().toFile(); SequenceManager.get().writeFasta(getContainer(), getUser(), _analysis.getSequencesView(), fastaFile); } @@ -261,11 +269,11 @@ private void sendFilesToGalaxy(GalaxyServer server) throws IOException, URISynta // in /matches into the data library. if (AppProps.getInstance().isDevMode()) { - File matchesDir = new File(_dir, "matches"); + FileLike matchesDir = _dir.resolveChild("matches"); if (matchesDir.exists()) { - File matchesFile = new File(matchesDir, GenotypingManager.MATCHES_FILE_NAME); + FileLike matchesFile = matchesDir.resolveChild(GenotypingManager.MATCHES_FILE_NAME); if (matchesFile.exists()) root.uploadFromImportDirectory(_dir.getName() + "/matches", "txt", null, true); diff --git a/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java b/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java index be949517..bc38199c 100644 --- a/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java +++ b/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java @@ -16,12 +16,13 @@ package org.labkey.genotyping.galaxy; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; import org.labkey.api.util.ContextListener; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.ShutdownListener; +import org.labkey.api.util.logging.LogHelper; import org.labkey.genotyping.GenotypingManager; +import org.labkey.vfs.FileLike; -import java.io.File; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -42,11 +43,11 @@ */ public class WorkflowCompletionMonitor implements ShutdownListener { - private static final Logger LOG = LogManager.getLogger(WorkflowCompletionMonitor.class); + private static final Logger LOG = LogHelper.getLogger(WorkflowCompletionMonitor.class, "Logger for Genotyping workflow completion monitor"); private static final WorkflowCompletionMonitor INSTANCE = new WorkflowCompletionMonitor(); private final ScheduledExecutorService _executor = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "Genotyping Workflow Completion Monitor")); - private final List _pendingCompletionFiles = new CopyOnWriteArrayList<>(); + private final List _pendingCompletionFiles = new CopyOnWriteArrayList<>(); static @@ -82,10 +83,10 @@ private WorkflowCompletionMonitor() } - public void monitor(File completionFile) + public void monitor(FileLike completionFile) { _pendingCompletionFiles.add(completionFile); - LOG.info("Monitoring for " + completionFile.getAbsolutePath()); + LOG.info("Monitoring for {}", FileUtil.getAbsolutePath(completionFile.toNioPathForRead())); } @@ -95,12 +96,6 @@ public String getName() return "Genotyping workflow completion monitor"; } - @Override - public void shutdownPre() - { - } - - @Override public void shutdownStarted() { @@ -117,35 +112,38 @@ public void run() if (size > 0) { - LOG.info("Checking for completion of " + size + " analys" + (1 == size ? "is" : "es")); + LOG.info("Checking for completion of {} analys{}", size, 1 == size ? "is" : "es"); - for (File file : _pendingCompletionFiles) + for (FileLike file : _pendingCompletionFiles) { if (file.exists()) { try { // Load analysis properties - Properties props = GenotypingManager.get().readProperties(file.getParentFile()); + Properties props = GenotypingManager.get().readProperties(file.getParent()); // POST to the provided URL to signal LabKey Server that the workflow is complete - String url = (String)props.get("url"); - String analysisId = (String)props.get("analysis"); - LOG.info("Detected completion file for analysis " + analysisId + "; attempting to signal LabKey Server at " + url); - HttpClient client = HttpClient.newHttpClient(); - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build(); - BodyHandler bodyHandler = HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8); - HttpResponse response = client.send(request, bodyHandler); - String message = response.body(); - - LOG.info("LabKey response to analysis " + analysisId + " completion: \"" + message + "\""); + String url = (String) props.get("url"); + String analysisId = (String) props.get("analysis"); + LOG.info("Detected completion file for analysis {}; attempting to signal LabKey Server at {}", analysisId, url); + + try (HttpClient client = HttpClient.newHttpClient()) + { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build(); + BodyHandler bodyHandler = HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8); + HttpResponse response = client.send(request, bodyHandler); + String message = response.body(); + + LOG.info("LabKey response to analysis {} completion: \"{}\"", analysisId, message); + } } catch (Throwable t) { - LOG.error("Exception while completing " + file.getAbsolutePath(), t); + LOG.error("Exception while completing {}", file.toNioPathForRead().toAbsolutePath(), t); } finally { From 17a93342f240290ce97698197abc138fd9603f8a Mon Sep 17 00:00:00 2001 From: ian Date: Thu, 27 Mar 2025 15:24:45 -0700 Subject: [PATCH 2/9] Fix run paths --- .../genotyping/GenotypingController.java | 52 ++++++++++++++----- .../labkey/genotyping/GenotypingManager.java | 2 +- .../org/labkey/genotyping/GenotypingRun.java | 25 ++++++--- .../labkey/genotyping/SubmitAnalysisJob.java | 2 +- 4 files changed, 59 insertions(+), 22 deletions(-) diff --git a/genotyping/src/org/labkey/genotyping/GenotypingController.java b/genotyping/src/org/labkey/genotyping/GenotypingController.java index e468f82b..8cb6efd7 100644 --- a/genotyping/src/org/labkey/genotyping/GenotypingController.java +++ b/genotyping/src/org/labkey/genotyping/GenotypingController.java @@ -125,6 +125,7 @@ import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; import java.nio.file.Paths; import java.sql.ResultSet; import java.sql.SQLException; @@ -1009,9 +1010,13 @@ private String importReads(ImportReadsForm form) throws Exception if (null == form.getPlatform()) return "You must specify a sequence platform"; + PipeRoot root = PipelineService.get().getPipelineRootSetting(getContainer()); + if (!root.getRootFileLike().isDescendant(FileUtil.createUri(form.getReadsPath()))) + return "File must be a descendant of a pipeline"; + try { - FileLike readsFile = new FileSystemLike.Builder(FileUtil.createUri(form.getReadsPath())).readonly().root(); + FileLike readsFile = root.resolvePathToFileLike(root.relativePath(Paths.get(form.getReadsPath()))); GenotypingRun run; try @@ -1027,7 +1032,6 @@ private String importReads(ImportReadsForm form) throws Exception } ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), getViewContext().getActionURL()); - PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); ImportReadsForm.Platforms platform = ImportReadsForm.Platforms.valueOf(form.getPlatform()); platform.prepareAndQueueRunJob(vbi, root, new File(form.getReadsPath()), run, form.getPrefix()); @@ -1182,10 +1186,18 @@ private ColumnInfo getColumnInfo(Map fieldMap, String colu public boolean handlePost(AnalyzeForm form, BindException errors) throws Exception { GenotypingRun run = GenotypingManager.get().getRun(getContainer(), form.getRun()); - FileLike readsPath = run.getWorkingDir().resolveChild(run.getFileName()); - ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), getViewContext().getActionURL()); + if (run == null) + { + errors.rejectValue("run", ERROR_MSG, "No run found"); + return false; + } + PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + FileLike readsPath = run.getWorkingDir(); + FileLike readsFile = readsPath.resolveChild(run.getFileName()); + ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), getViewContext().getActionURL()); + String sequencesViewName = form.getSequencesView(); String description = form.getDescription(); String sequencesView = DEFAULT_VIEW_PLACEHOLDER.equals(sequencesViewName) ? null : sequencesViewName; @@ -1206,7 +1218,7 @@ public boolean handlePost(AnalyzeForm form, BindException errors) throws Excepti GenotypingAnalysis analysis = GenotypingManager.get().createAnalysis(getContainer(), getUser(), run, description, sequencesView); try { - PipelineJob analysisJob = new SubmitAnalysisJob(vbi, root, readsPath, analysis, sampleKeys); + PipelineJob analysisJob = new SubmitAnalysisJob(vbi, root, readsFile, analysis, sampleKeys); PipelineService.get().queueJob(analysisJob); } catch (MinorConfigurationException e) @@ -1296,8 +1308,12 @@ public Object execute(ImportAnalysisForm form, BindException errors) throws Exce try { + PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + if (!root.getRootFileLike().isDescendant(FileUtil.createUri(form.getPath()))) + throw new FileNotFoundException("Import folder not found." + form.getPath()); + int analysisId = form.getAnalysis(); - FileLike analysisDir = new FileSystemLike.Builder(Paths.get(form.getPath())).readwrite().root(); + FileLike analysisDir = root.resolvePathToFileLike(form.getPath()); User user = getUser(); @@ -1383,16 +1399,26 @@ public void validateCommand(PipelinePathForm target, Errors errors) @Override public boolean handlePost(PipelinePathForm form, BindException errors) throws IOException, PipelineValidationException { + Container container = getContainer(); // Manual upload of genotyping analysis; pipeline provider posts to this action with matches file. - FileLike matches = FileSystemLike.wrapFile(form.getValidatedSingleFile(getContainer())); - FileLike analysisDir = matches.getParent(); + Path singleFile = form.getValidatedSinglePath(container); - // Load properties to determine the run. - Properties props = GenotypingManager.get().readProperties(analysisDir); - int analysisId = Integer.parseInt((String)props.get("analysis")); - importAnalysis(analysisId, analysisDir, getUser()); + if (form.getPipeRoot(container).getRootFileLike().isDescendant(singleFile.toUri())) + { - return true; + FileLike matches = form.getPipeRoot(container).resolvePathToFileLike(singleFile.toString()); + FileLike analysisDir = matches.getParent(); + + // Load properties to determine the run. + Properties props = GenotypingManager.get().readProperties(analysisDir); + int analysisId = Integer.parseInt((String) props.get("analysis")); + importAnalysis(analysisId, analysisDir, getUser()); + + return true; + } + + errors.reject(ERROR_MSG, "File was not found under the pipeline root"); + return false; } @Override diff --git a/genotyping/src/org/labkey/genotyping/GenotypingManager.java b/genotyping/src/org/labkey/genotyping/GenotypingManager.java index 4bba3670..8f8e2bc6 100644 --- a/genotyping/src/org/labkey/genotyping/GenotypingManager.java +++ b/genotyping/src/org/labkey/genotyping/GenotypingManager.java @@ -130,7 +130,7 @@ public GenotypingRun createRun(Container c, User user, Integer metaDataId, FileL if (null != metaDataId) mdRun = getMetaDataRun(c, user, metaDataId, "importing reads"); - GenotypingRun run = new GenotypingRun(c, readsFile, mdRun, platform); + GenotypingRun run = new GenotypingRun(c, readsFile.toNioPathForWrite().toFile(), mdRun, platform); return Table.insert(user, GenotypingSchema.get().getRunsTable(), run); } diff --git a/genotyping/src/org/labkey/genotyping/GenotypingRun.java b/genotyping/src/org/labkey/genotyping/GenotypingRun.java index 8db217cd..895ea1ad 100644 --- a/genotyping/src/org/labkey/genotyping/GenotypingRun.java +++ b/genotyping/src/org/labkey/genotyping/GenotypingRun.java @@ -15,12 +15,17 @@ */ package org.labkey.genotyping; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.MemTracker; import org.labkey.vfs.FileLike; +import java.io.File; +import java.nio.file.Paths; import java.util.Date; /** @@ -34,7 +39,7 @@ public class GenotypingRun private Container _container; private int _createdBy; private Date _created; - private FileLike _workingDir; + private String _path; private String _platform; private String _fileName; private Integer _metaDataId = null; @@ -45,11 +50,11 @@ public GenotypingRun() MemTracker.getInstance().put(this); } - public GenotypingRun(Container c, FileLike readsFile, @Nullable MetaDataRun metaDataRun, String platform) + public GenotypingRun(Container c, File readsFile, @Nullable MetaDataRun metaDataRun, String platform) { this(); setContainer(c); - setWorkingDir(readsFile.getParent()); + setPath(FileUtil.getAbsoluteCaseSensitiveFile(readsFile.getParentFile()).getPath()); setFileName(readsFile.getName()); setPlatform(platform); @@ -117,14 +122,14 @@ public void setCreated(Date created) _created = created; } - public FileLike getWorkingDir() + public String getPath() { - return _workingDir; + return _path; } - public void setWorkingDir(FileLike path) + public void setPath(String path) { - _workingDir = path; + _path = path; } public String getPlatform() @@ -166,4 +171,10 @@ public void setStatusEnum(Status statusEnum) { _status = statusEnum.getStatusId(); } + + @JsonIgnore + public FileLike getWorkingDir() + { + return PipelineService.get().findPipelineRoot(getContainer()).resolvePathToFileLike(PipelineService.get().findPipelineRoot(getContainer()).relativePath(Paths.get(getPath()))); + } } diff --git a/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java b/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java index b7e822b1..61763255 100644 --- a/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java +++ b/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java @@ -113,7 +113,7 @@ public SubmitAnalysisJob(ViewBackgroundInfo info, PipeRoot root, FileLike reads, setLogFile(_analysisDir.resolveChild(FileUtil.makeFileNameWithTimestamp("submit_analysis", "log")).toNioPathForWrite()); info("Creating analysis directory: " + _analysisDir.getName()); - _analysis.setPath(_analysisDir.getPath().toString()); + _analysis.setPath(FileUtil.getAbsolutePath(_analysisDir.toNioPathForRead())); _analysis.setFileName(_analysisDir.getName()); Table.update(getUser(), GenotypingSchema.get().getAnalysesTable(), PageFlowUtil.map("Path", _analysis.getPath(), "FileName", _analysis.getFileName()), _analysis.getRowId()); } From daf7448141f8608ee35973c09ab1abbcab4c2bfa Mon Sep 17 00:00:00 2001 From: ian Date: Tue, 8 Apr 2025 14:37:14 -0700 Subject: [PATCH 3/9] remove unused variable --- genotyping/src/org/labkey/genotyping/GenotypingController.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/genotyping/src/org/labkey/genotyping/GenotypingController.java b/genotyping/src/org/labkey/genotyping/GenotypingController.java index 8cb6efd7..ec90f441 100644 --- a/genotyping/src/org/labkey/genotyping/GenotypingController.java +++ b/genotyping/src/org/labkey/genotyping/GenotypingController.java @@ -1194,8 +1194,7 @@ public boolean handlePost(AnalyzeForm form, BindException errors) throws Excepti PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); - FileLike readsPath = run.getWorkingDir(); - FileLike readsFile = readsPath.resolveChild(run.getFileName()); + FileLike readsFile = run.getWorkingDir().resolveChild(run.getFileName()); ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), getViewContext().getActionURL()); String sequencesViewName = form.getSequencesView(); From eada6538a271724ed59cb1deb1702794227a5871 Mon Sep 17 00:00:00 2001 From: ian Date: Tue, 8 Apr 2025 14:47:01 -0700 Subject: [PATCH 4/9] fixing a compilation error that I don't understand since it implements the interface... --- .../genotyping/galaxy/WorkflowCompletionMonitor.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java b/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java index bc38199c..73b8a637 100644 --- a/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java +++ b/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java @@ -128,8 +128,10 @@ public void run() String analysisId = (String) props.get("analysis"); LOG.info("Detected completion file for analysis {}; attempting to signal LabKey Server at {}", analysisId, url); - try (HttpClient client = HttpClient.newHttpClient()) + HttpClient client = null; + try { + client = HttpClient.newHttpClient(); HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(url)) .POST(HttpRequest.BodyPublishers.noBody()) @@ -140,6 +142,12 @@ public void run() LOG.info("LabKey response to analysis {} completion: \"{}\"", analysisId, message); } + finally + { + // Unclear to me why this can't be used with the try-with-resources... + if (client != null) + client.close(); + } } catch (Throwable t) { From ca68d86133a3fdbe535b4f0ef43347eb621a21e3 Mon Sep 17 00:00:00 2001 From: ian Date: Thu, 17 Apr 2025 15:15:43 -0700 Subject: [PATCH 5/9] TC java version doesn't have a close method --- .../galaxy/WorkflowCompletionMonitor.java | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java b/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java index 73b8a637..e7a4f048 100644 --- a/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java +++ b/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java @@ -128,26 +128,18 @@ public void run() String analysisId = (String) props.get("analysis"); LOG.info("Detected completion file for analysis {}; attempting to signal LabKey Server at {}", analysisId, url); - HttpClient client = null; - try - { - client = HttpClient.newHttpClient(); - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build(); - BodyHandler bodyHandler = HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8); - HttpResponse response = client.send(request, bodyHandler); - String message = response.body(); - - LOG.info("LabKey response to analysis {} completion: \"{}\"", analysisId, message); - } - finally - { - // Unclear to me why this can't be used with the try-with-resources... - if (client != null) - client.close(); - } + // TODO: with Java 21 this should be moved to an autoclosable + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build(); + BodyHandler bodyHandler = HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8); + HttpResponse response = client.send(request, bodyHandler); + String message = response.body(); + + LOG.info("LabKey response to analysis {} completion: \"{}\"", analysisId, message); + } catch (Throwable t) { From e9f510c9b06a0f2a674c15d9c4d493ef8b4120e9 Mon Sep 17 00:00:00 2001 From: ian Date: Mon, 21 Apr 2025 16:19:06 -0700 Subject: [PATCH 6/9] fix analysis dir path --- .../src/org/labkey/genotyping/GenotypingController.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/genotyping/src/org/labkey/genotyping/GenotypingController.java b/genotyping/src/org/labkey/genotyping/GenotypingController.java index ec90f441..924762ae 100644 --- a/genotyping/src/org/labkey/genotyping/GenotypingController.java +++ b/genotyping/src/org/labkey/genotyping/GenotypingController.java @@ -1307,15 +1307,10 @@ public Object execute(ImportAnalysisForm form, BindException errors) throws Exce try { - PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); - if (!root.getRootFileLike().isDescendant(FileUtil.createUri(form.getPath()))) - throw new FileNotFoundException("Import folder not found." + form.getPath()); - + FileLike analysisDir = FileSystemLike.getVerifiedFileLike(getContainer(), form.getPath()); int analysisId = form.getAnalysis(); - FileLike analysisDir = root.resolvePathToFileLike(form.getPath()); User user = getUser(); - if (user.isGuest()) { Properties props = GenotypingManager.get().readProperties(analysisDir); From bb1e5e36de57e2aee86a8d600511d045ded59bcb Mon Sep 17 00:00:00 2001 From: ian Date: Tue, 22 Apr 2025 09:35:52 -0700 Subject: [PATCH 7/9] Fix lint warning --- genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java b/genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java index 02d165e1..95cc0666 100644 --- a/genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java +++ b/genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java @@ -69,7 +69,7 @@ public ImportPacBioReadsJob(ViewBackgroundInfo info, PipeRoot root, File sampleF super(ImportPacBioReadsPipelineProvider.NAME, info, root, run); _sampleFile = sampleFile; _fastqPrefix = fastqPrefix; - setLogFile(new File(_sampleFile.getParentFile(), FileUtil.makeFileNameWithTimestamp("import_pacbio_reads", "log"))); + setLogFile(FileUtil.appendName(_sampleFile.getAbsoluteFile(), FileUtil.makeFileNameWithTimestamp("import_pacbio_reads", "log"))); } @Override From f6988fb0851e29493ed677f2c93012ff1f35f7b7 Mon Sep 17 00:00:00 2001 From: ian Date: Tue, 22 Apr 2025 15:29:13 -0700 Subject: [PATCH 8/9] Fix writer issue... --- genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java | 4 ++-- .../labkey/genotyping/galaxy/WorkflowCompletionMonitor.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java b/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java index 61763255..39988a45 100644 --- a/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java +++ b/genotyping/src/org/labkey/genotyping/SubmitAnalysisJob.java @@ -207,9 +207,9 @@ protected int write() return rows.getValue(); } - }; OutputStream os = _analysisDir.resolveChild("reads.txt").openOutputStream()) + }) { - writer.write(os); + writer.write(_analysisDir.resolveChild("reads.txt").toNioPathForWrite().toFile()); } } diff --git a/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java b/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java index e7a4f048..0abbd0a3 100644 --- a/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java +++ b/genotyping/src/org/labkey/genotyping/galaxy/WorkflowCompletionMonitor.java @@ -114,6 +114,9 @@ public void run() { LOG.info("Checking for completion of {} analys{}", size, 1 == size ? "is" : "es"); + // TODO: with Java 21 this should be moved to an autoclosable + HttpClient client = HttpClient.newHttpClient(); + for (FileLike file : _pendingCompletionFiles) { if (file.exists()) @@ -128,8 +131,6 @@ public void run() String analysisId = (String) props.get("analysis"); LOG.info("Detected completion file for analysis {}; attempting to signal LabKey Server at {}", analysisId, url); - // TODO: with Java 21 this should be moved to an autoclosable - HttpClient client = HttpClient.newHttpClient(); HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(url)) .POST(HttpRequest.BodyPublishers.noBody()) @@ -139,7 +140,6 @@ public void run() String message = response.body(); LOG.info("LabKey response to analysis {} completion: \"{}\"", analysisId, message); - } catch (Throwable t) { From 5e44bb2d79f54dc766b0489e669631421e3e2efe Mon Sep 17 00:00:00 2001 From: ian Date: Tue, 22 Apr 2025 16:34:05 -0700 Subject: [PATCH 9/9] Fix log naming error and some lint warnings. --- .../labkey/genotyping/Import454ReadsJob.java | 7 ++++-- .../genotyping/ImportIlluminaReadsJob.java | 7 ++++-- .../genotyping/ImportPacBioReadsJob.java | 25 +++++++++++-------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/genotyping/src/org/labkey/genotyping/Import454ReadsJob.java b/genotyping/src/org/labkey/genotyping/Import454ReadsJob.java index 39d3301a..c138a120 100644 --- a/genotyping/src/org/labkey/genotyping/Import454ReadsJob.java +++ b/genotyping/src/org/labkey/genotyping/Import454ReadsJob.java @@ -30,6 +30,8 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.util.Formats; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -65,8 +67,9 @@ protected Import454ReadsJob(@JsonProperty("_reads") File reads) public Import454ReadsJob(ViewBackgroundInfo info, PipeRoot root, File reads, GenotypingRun run) { super(Import454ReadsPipelineProvider.NAME, info, root, run); - _reads = reads; - setLogFile(new File(_reads.getParentFile(), FileUtil.makeFileNameWithTimestamp("import_reads", "log"))); + FileLike verifiedFileLike = FileSystemLike.getVerifiedFileLike(root.getContainer(), reads.getAbsolutePath()); + _reads = FileSystemLike.toFile(verifiedFileLike); + setLogFile(verifiedFileLike.getParent().resolveChild(FileUtil.makeFileNameWithTimestamp("import_reads", "log")).toNioPathForWrite()); } @Override diff --git a/genotyping/src/org/labkey/genotyping/ImportIlluminaReadsJob.java b/genotyping/src/org/labkey/genotyping/ImportIlluminaReadsJob.java index dfa93f1c..b4b5b1ea 100644 --- a/genotyping/src/org/labkey/genotyping/ImportIlluminaReadsJob.java +++ b/genotyping/src/org/labkey/genotyping/ImportIlluminaReadsJob.java @@ -34,6 +34,8 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.util.Pair; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -71,9 +73,10 @@ protected ImportIlluminaReadsJob(@JsonProperty("_sampleFile") File sampleFile, @ public ImportIlluminaReadsJob(ViewBackgroundInfo info, PipeRoot root, File sampleFile, GenotypingRun run, @Nullable String fastqPrefix) { super(ImportIlluminaReadsPipelineProvider.NAME, info, root, run); - _sampleFile = sampleFile; + FileLike verifiedFileLike = FileSystemLike.getVerifiedFileLike(root.getContainer(), sampleFile.getAbsolutePath()); + _sampleFile = FileSystemLike.toFile(verifiedFileLike); _fastqPrefix = fastqPrefix; - setLogFile(new File(_sampleFile.getParentFile(), FileUtil.makeFileNameWithTimestamp("import_reads", "log"))); + setLogFile(verifiedFileLike.getParent().resolveChild(FileUtil.makeFileNameWithTimestamp("import_reads", "log")).toNioPathForWrite()); } @Override diff --git a/genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java b/genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java index 95cc0666..18b2507e 100644 --- a/genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java +++ b/genotyping/src/org/labkey/genotyping/ImportPacBioReadsJob.java @@ -39,6 +39,8 @@ import org.labkey.api.reader.Readers; import org.labkey.api.util.FileUtil; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.FileNotFoundException; @@ -51,6 +53,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; public class ImportPacBioReadsJob extends ReadsJob @@ -67,9 +70,10 @@ protected ImportPacBioReadsJob() {} public ImportPacBioReadsJob(ViewBackgroundInfo info, PipeRoot root, File sampleFile, GenotypingRun run, @Nullable String fastqPrefix) { super(ImportPacBioReadsPipelineProvider.NAME, info, root, run); - _sampleFile = sampleFile; + FileLike verifiedFileLike = FileSystemLike.getVerifiedFileLike(root.getContainer(), sampleFile.getAbsolutePath()); + _sampleFile = FileSystemLike.toFile(verifiedFileLike); _fastqPrefix = fastqPrefix; - setLogFile(FileUtil.appendName(_sampleFile.getAbsoluteFile(), FileUtil.makeFileNameWithTimestamp("import_pacbio_reads", "log"))); + setLogFile(verifiedFileLike.getParent().resolveChild(FileUtil.makeFileNameWithTimestamp("import_pacbio_reads", "log")).toNioPathForWrite()); } @Override @@ -117,7 +121,7 @@ private void importReads() throws PipelineJobException, SQLException collectFastqFilesAsPools(); //error if no pools were found and no fastq files were found to parse - if (_pools.size() == 0) + if (_pools.isEmpty()) getLogger().warn("No pools/FASTQ files" + (_fastqPrefix == null ? "" : " matching the prefix '" + _fastqPrefix + "'") + " were found. Check that files are under a 'poolX' directory"); persistPacBioPoolRecords(sampleNameSampleIdMap); @@ -193,7 +197,7 @@ private void collectFastqFilesAsPools() if (_fastqPrefix != null)//filter files w/prefix (if prefix is provided) { - for (File f : sampleSheetRootFolder.listFiles()) + for (File f : Objects.requireNonNull(sampleSheetRootFolder.listFiles())) { Collection filteredWithPrefixFastqFiles = new LinkedList<>(); @@ -264,15 +268,15 @@ private void persistPacBioPoolRecords(Map sampleNameSampleIdMap private int extractPoolNumFromDirectoryName(String dirName) { - String[] splitStr = dirName.split("_"); //expected directory name pattern to be "pool1_barcoded_fastq", "123_pool2_..." + String[] splitStr = dirName.split("_"); //expected the directory name pattern to be "pool1_barcoded_fastq", "123_pool2_..." //extract pool num - for(int i = 0 ; i < splitStr.length ; i++) + for (String s : splitStr) { - if (splitStr[i].contains(_dirSubstring)) - return Integer.valueOf(splitStr[i].substring(_dirSubstring.length())); + if (s.contains(_dirSubstring)) + return Integer.valueOf(s.substring(_dirSubstring.length())); } - return 1; //if pool num is not found, then default pool num to be 1. + return 1; //if the pool num is not found, then the default pool num is 1. } private int getSampleId(File file, Map sampleNameSampleIdMap) @@ -292,7 +296,6 @@ private int getSampleId(File file, Map sampleNameSampleIdMap) public Integer getNumReads(File fastqFile, int fileNum, int poolNum) { - FastqReader reader = null; int totalReads = 0; setStatus("PARSING FILE " + fileNum + " in Pool " + poolNum); @@ -305,7 +308,7 @@ public Integer getNumReads(File fastqFile, int fileNum, int poolNum) _logger.info("Beginning to parse file: " + fastqFile.getName()); - reader = new FastqReader(fastqFile); + FastqReader reader = new FastqReader(fastqFile); while (reader.hasNext()) { try