diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java index dfc4f0232..27152ed45 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java @@ -44,6 +44,7 @@ abstract public class AbstractCommandWrapper implements CommandWrapper private File _outputDir = null; private File _workingDir = null; private Logger _log; + private boolean _logPath = false; private Level _logLevel = Level.DEBUG; private boolean _warnNonZeroExits = true; private boolean _throwNonZeroExits = true; @@ -205,9 +206,11 @@ private void setPath(ProcessBuilder pb) { String path = System.getenv("PATH"); - getLogger().debug("Existing PATH: " + path); - getLogger().debug("toolDir: " + toolDir); - + if (_logPath) + { + getLogger().debug("Existing PATH: " + path); + getLogger().debug("toolDir: " + toolDir); + } if (path == null) { @@ -229,11 +232,19 @@ private void setPath(ProcessBuilder pb) path = fileExe.getParent() + File.pathSeparatorChar + path; } - getLogger().debug("using path: " + path); + if (_logPath) + { + getLogger().debug("using path: " + path); + } pb.environment().put("PATH", path); } } + public void setLogPath(boolean logPath) + { + _logPath = logPath; + } + public void setOutputDir(File outputDir) { _outputDir = outputDir; diff --git a/SequenceAnalysis/pipeline_code/extra_tools_install.sh b/SequenceAnalysis/pipeline_code/extra_tools_install.sh index e3cce3c55..8ecd60f37 100755 --- a/SequenceAnalysis/pipeline_code/extra_tools_install.sh +++ b/SequenceAnalysis/pipeline_code/extra_tools_install.sh @@ -319,3 +319,17 @@ then else echo "Already installed" fi + +if [[ ! -e ${LKTOOLS_DIR}/sawfish || ! -z $FORCE_REINSTALL ]]; +then + echo "Cleaning up previous installs" + rm -Rf $LKTOOLS_DIR/sawfish* + + wget https://github.com/PacificBiosciences/sawfish/releases/download/v2.0.0/sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz + tar -xzf sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz + + mv sawfish-v2.0.0-x86_64-unknown-linux-gnu $LKTOOLS_DIR/ + ln -s $LKTOOLS_DIR/sawfish-v2.0.0/bin/sawfish $LKTOOLS_DIR/ +else + echo "Already installed" +fi diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java index 94f2bde2e..3ac54c27c 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java @@ -123,6 +123,8 @@ import org.labkey.sequenceanalysis.run.analysis.PbsvAnalysis; import org.labkey.sequenceanalysis.run.analysis.PbsvJointCallingHandler; import org.labkey.sequenceanalysis.run.analysis.PindelAnalysis; +import org.labkey.sequenceanalysis.run.analysis.SawfishAnalysis; +import org.labkey.sequenceanalysis.run.analysis.SawfishJointCallingHandler; import org.labkey.sequenceanalysis.run.analysis.SequenceBasedTypingAnalysis; import org.labkey.sequenceanalysis.run.analysis.SnpCountAnalysis; import org.labkey.sequenceanalysis.run.analysis.SubreadAnalysis; @@ -342,6 +344,7 @@ public static void registerPipelineSteps() SequencePipelineService.get().registerPipelineStep(new PindelAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new PbsvAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new GenrichStep.Provider()); + SequencePipelineService.get().registerPipelineStep(new SawfishAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new PARalyzerAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new RnaSeQCStep.Provider()); @@ -400,6 +403,7 @@ public static void registerPipelineSteps() SequenceAnalysisService.get().registerFileHandler(new NextCladeHandler()); SequenceAnalysisService.get().registerFileHandler(new ConvertToCramHandler()); SequenceAnalysisService.get().registerFileHandler(new PbsvJointCallingHandler()); + SequenceAnalysisService.get().registerFileHandler(new SawfishJointCallingHandler()); SequenceAnalysisService.get().registerFileHandler(new DeepVariantHandler()); SequenceAnalysisService.get().registerFileHandler(new GLNexusHandler()); SequenceAnalysisService.get().registerFileHandler(new ParagraphStep()); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java new file mode 100644 index 000000000..8039ff338 --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java @@ -0,0 +1,105 @@ +package org.labkey.sequenceanalysis.run.analysis; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.sequenceanalysis.model.AnalysisModel; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.sequenceanalysis.pipeline.AbstractAnalysisStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStep; +import org.labkey.api.sequenceanalysis.pipeline.AnalysisOutputImpl; +import org.labkey.api.sequenceanalysis.pipeline.AnalysisStep; +import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; +import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SamtoolsIndexer; +import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner; +import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.sequenceanalysis.util.SequenceUtil; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class SawfishAnalysis extends AbstractPipelineStep implements AnalysisStep +{ + public SawfishAnalysis(PipelineStepProvider provider, PipelineContext ctx) + { + super(provider, ctx); + } + + public static class Provider extends AbstractAnalysisStepProvider + { + public Provider() + { + super("sawfish", "Sawfish Analysis", null, "This will run sawfish SV dicvoery and calling on the selected BAMs", List.of(), null, null); + } + + + @Override + public SawfishAnalysis create(PipelineContext ctx) + { + return new SawfishAnalysis(this, ctx); + } + } + + @Override + public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException + { + AnalysisOutputImpl output = new AnalysisOutputImpl(); + + List args = new ArrayList<>(); + args.add(getExe().getPath()); + args.add("discover"); + + args.add("--bam"); + args.add(inputBam.getPath()); + + // NOTE: sawfish stores the absolute path of the FASTA in the output JSON, so dont rely on working copies: + args.add("--ref"); + args.add(referenceGenome.getSourceFastaFile().getPath()); + + File svOutDir = new File(outputDir, "sawfish"); + args.add("--output-dir"); + args.add(svOutDir.getPath()); + + Integer maxThreads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger()); + if (maxThreads != null) + { + args.add("--threads"); + args.add(String.valueOf(maxThreads)); + } + + File bcf = new File(svOutDir, "candidate.sv.bcf"); + File bcfIdx = new File(bcf.getPath() + ".csi"); + if (bcfIdx.exists()) + { + getPipelineCtx().getLogger().debug("BCF index already exists, reusing output"); + } + else + { + new SimpleScriptWrapper(getPipelineCtx().getLogger()).execute(args); + } + + if (!bcf.exists()) + { + throw new PipelineJobException("Unable to find file: " + bcf.getPath()); + } + + output.addSequenceOutput(bcf, rs.getName() + ": sawfish", "Sawfish SV Discovery", rs.getReadsetId(), null, referenceGenome.getGenomeId(), null); + + return output; + } + + @Override + public Output performAnalysisPerSampleLocal(AnalysisModel model, File inputBam, File referenceFasta, File outDir) throws PipelineJobException + { + return null; + } + + private File getExe() + { + return SequencePipelineService.get().getExeForPackage("SAWFISHPATH", "sawfish"); + } +} \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java new file mode 100644 index 000000000..9beae27c6 --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java @@ -0,0 +1,180 @@ +package org.labkey.sequenceanalysis.run.analysis; + +import org.apache.commons.io.FileUtils; +import org.json.JSONObject; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.SequenceOutputFile; +import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; +import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.sequenceanalysis.SequenceAnalysisModule; +import org.labkey.sequenceanalysis.util.SequenceUtil; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.stream.Collectors; + +public class SawfishJointCallingHandler extends AbstractParameterizedOutputHandler +{ + private static final String OUTPUT_CATEGORY = "Sawfish VCF"; + + public SawfishJointCallingHandler() + { + super(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.NAME), "Sawfish Joint-Call", "Runs sawfish joint-call, which jointly calls SVs from PacBio CCS data", new LinkedHashSet<>(List.of("sequenceanalysis/panel/VariantScatterGatherPanel.js")), Arrays.asList( + ToolParameterDescriptor.create("fileName", "VCF Filename", "The name of the resulting file.", "textfield", new JSONObject(){{ + put("allowBlank", false); + put("doNotIncludeInTemplates", true); + }}, null) + )); + } + + @Override + public boolean canProcess(SequenceOutputFile o) + { + return o.getFile() != null && SequenceUtil.FILETYPE.bcf.getFileType().isType(o.getFile()); + } + + @Override + public boolean doRunRemote() + { + return true; + } + + @Override + public boolean doRunLocal() + { + return false; + } + + @Override + public SequenceOutputProcessor getProcessor() + { + return new Processor(); + } + + public static class Processor implements SequenceOutputProcessor + { + @Override + public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + { + + } + + @Override + public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException + { + List filesToProcess = inputFiles.stream().map(SequenceOutputFile::getFile).collect(Collectors.toList()); + + ReferenceGenome genome = ctx.getSequenceSupport().getCachedGenomes().iterator().next(); + String outputBaseName = ctx.getParams().getString("fileName"); + if (!outputBaseName.toLowerCase().endsWith(".gz")) + { + outputBaseName = outputBaseName.replaceAll(".gz$", ""); + } + + if (!outputBaseName.toLowerCase().endsWith(".vcf")) + { + outputBaseName = outputBaseName.replaceAll(".vcf$", ""); + } + + File expectedFinalOutput = new File(ctx.getOutputDir(), outputBaseName + ".vcf.gz"); + + File ouputVcf = runSawfishCall(ctx, filesToProcess, genome, outputBaseName); + + SequenceOutputFile so = new SequenceOutputFile(); + so.setName("Sawfish call: " + outputBaseName); + so.setFile(ouputVcf); + so.setCategory(OUTPUT_CATEGORY); + so.setLibrary_id(genome.getGenomeId()); + + ctx.addSequenceOutput(so); + } + + private File runSawfishCall(JobContext ctx, List inputs, ReferenceGenome genome, String outputBaseName) throws PipelineJobException + { + if (inputs.isEmpty()) + { + throw new PipelineJobException("No inputs provided"); + } + + List args = new ArrayList<>(); + args.add(getExe().getPath()); + args.add("joint-call"); + + Integer maxThreads = SequencePipelineService.get().getMaxThreads(ctx.getLogger()); + if (maxThreads != null) + { + args.add("--threads"); + args.add(String.valueOf(maxThreads)); + } + + for (File sample : inputs) + { + args.add("--sample"); + args.add(sample.getParentFile().getPath()); + } + + File outDir = new File(ctx.getOutputDir(), "sawfish"); + args.add("--output-dir"); + args.add(outDir.getPath()); + + new SimpleScriptWrapper(ctx.getLogger()).execute(args); + + File vcfOut = new File(outDir, "genotyped.sv.vcf.gz"); + if (!vcfOut.exists()) + { + throw new PipelineJobException("Unable to find file: " + vcfOut.getPath()); + } + + File vcfOutFinal = new File(ctx.getOutputDir(), outputBaseName + ".vcf.gz"); + + try + { + if (vcfOutFinal.exists()) + { + vcfOutFinal.delete(); + } + FileUtils.moveFile(vcfOut, vcfOutFinal); + + File targetIndex = new File(vcfOutFinal.getPath() + ".tbi"); + if (targetIndex.exists()) + { + targetIndex.delete(); + } + + File origIndex = new File(vcfOut.getPath() + ".tbi"); + if (origIndex.exists()) + { + FileUtils.moveFile(origIndex, targetIndex); + } + else + { + SequenceAnalysisService.get().ensureVcfIndex(vcfOutFinal, ctx.getLogger(), true); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + return vcfOutFinal; + } + + private File getExe() + { + return SequencePipelineService.get().getExeForPackage("SAWFISHPATH", "sawfish"); + } + } +} diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CramToBamWrapper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CramToBamWrapper.java new file mode 100644 index 000000000..1e6cf749d --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CramToBamWrapper.java @@ -0,0 +1,47 @@ +package org.labkey.sequenceanalysis.run.util; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class CramToBamWrapper extends SamtoolsRunner +{ + public CramToBamWrapper(Logger log) + { + super(log); + } + + public void convert(File inputCram, File outputBam, File fasta, @Nullable Integer threads) throws PipelineJobException + { + getLogger().info("Converting CRAM to BAM"); + + execute(getParams(inputCram, outputBam, fasta, threads)); + } + + private List getParams(File inputCram, File outputBam, File fasta, @Nullable Integer threads) + { + List params = new ArrayList<>(); + params.add(getSamtoolsPath().getPath()); + params.add("view"); + params.add("-b"); + params.add("-T"); + params.add(fasta.getPath()); + params.add("-o"); + params.add(outputBam.getPath()); + + if (threads != null) + { + params.add("-@"); + params.add(String.valueOf(threads)); + } + + params.add(inputCram.getPath()); + + return params; + } +} diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java index 0a2ce0784..a5357d176 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java @@ -96,6 +96,7 @@ public enum FILETYPE bed(Collections.singletonList(".bed"), true), bw(Collections.singletonList(".bw"), false), vcf(List.of(".vcf"), true), + bcf(List.of(".bcf"), true), gvcf(List.of(".g.vcf"), true); private final List _extensions; diff --git a/Studies/api-src/org/labkey/api/studies/study/StudyDefinition.java b/Studies/api-src/org/labkey/api/studies/study/StudyDefinition.java new file mode 100644 index 000000000..cff4174ba --- /dev/null +++ b/Studies/api-src/org/labkey/api/studies/study/StudyDefinition.java @@ -0,0 +1,669 @@ +package org.labkey.api.studies.study; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import org.json.JSONObject; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class StudyDefinition +{ + private Integer _rowId; + private String _studyName; + private String _label; + private String _category; + private String _description; + + private String _container; + private Integer _createdBy; + private Date _created; + private Integer _modifiedBy; + private Date _modified; + + private List _cohorts; + private List _anchorEvents; + private List _timepoints; + + public StudyDefinition() + { + + } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public String getStudyName() + { + return _studyName; + } + + public void setStudyName(String studyName) + { + _studyName = studyName; + } + + public String getLabel() + { + return _label; + } + + public void setLabel(String label) + { + _label = label; + } + + public String getCategory() + { + return _category; + } + + public void setCategory(String category) + { + _category = category; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } + + public Integer getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(Integer createdBy) + { + _createdBy = createdBy; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public Integer getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(Integer modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + + public List getCohorts() + { + return _cohorts; + } + + public void setCohorts(List cohorts) + { + _cohorts = cohorts; + } + + public List getAnchorEvents() + { + return _anchorEvents; + } + + public void setAnchorEvents(List anchorEvents) + { + _anchorEvents = anchorEvents; + } + + public List getTimepoints() + { + return _timepoints; + } + + public void setTimepoints(List timepoints) + { + _timepoints = timepoints; + } + + public static class StudyCohort + { + private Integer _rowId; + private Integer _studyId; + + private String _cohortName; + private String _label; + private String _category; + private String _description; + private Boolean _isControlGroup = false; + private Integer _sortOrder; + + private String _container; + private Integer _createdBy; + private Date _created; + + private Integer _modifiedBy; + private Date _modified; + + public StudyCohort() + { + + } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public Integer getStudyId() + { + return _studyId; + } + + public void setStudyId(Integer studyId) + { + _studyId = studyId; + } + + public String getCohortName() + { + return _cohortName; + } + + public void setCohortName(String cohortName) + { + _cohortName = cohortName; + } + + public String getLabel() + { + return _label; + } + + public void setLabel(String label) + { + _label = label; + } + + public String getCategory() + { + return _category; + } + + public void setCategory(String category) + { + _category = category; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public Boolean getIsControlGroup() + { + return _isControlGroup; + } + + public void setIsControlGroup(Boolean controlGroup) + { + _isControlGroup = controlGroup; + } + + public Integer getSortOrder() + { + return _sortOrder; + } + + public void setSortOrder(Integer sortOrder) + { + _sortOrder = sortOrder; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } + + public Integer getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(Integer createdBy) + { + _createdBy = createdBy; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public Integer getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(Integer modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + } + + public static class AnchorEvent + { + private Integer _rowId; + private Integer _studyId; + + private String _label; + private String _description; + private String _eventProviderName; + + private String _container; + private Integer _createdBy; + private Date _created; + + private Integer _modifiedBy; + private Date _modified; + + public AnchorEvent() + { + + } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public Integer getStudyId() + { + return _studyId; + } + + public void setStudyId(Integer studyId) + { + _studyId = studyId; + } + + public String getLabel() + { + return _label; + } + + public void setLabel(String label) + { + _label = label; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public String getEventProviderName() + { + return _eventProviderName; + } + + public void setEventProviderName(String eventProviderName) + { + _eventProviderName = eventProviderName; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } + + public Integer getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(Integer createdBy) + { + _createdBy = createdBy; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public Integer getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(Integer modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + } + + public static class Timepoint + { + private Integer _rowId; + private Integer _studyId; + private Integer _cohortId; + private String _label; + private String _labelShort; + private String _description; + + @JsonIgnore + private Integer _anchorEvent; + + @JsonIgnore + private String _anchorEventLabel; + + private String _cohortName; + private Integer _rangeMin; + private Integer _rangeMax; + + private String _container; + private Integer _createdBy; + private Date _created; + + private Integer _modifiedBy; + private Date _modified; + + public Timepoint() + { + + } + + // When reading from a JSON object, store the anchorEvent label + @JsonSetter("anchorEvent") + void readAnchorEvent(String lbl) { _anchorEventLabel = lbl; } + + @JsonGetter("anchorEvent") + String writeAnchorEvent() { return _anchorEventLabel; } + + // Call this to translate from label to index in order to fit the DB schema + void resolveAnchorEvent(Map idxByLabel) + { + Integer idx = idxByLabel.get(_anchorEventLabel); + if (idx == null) + throw new IllegalArgumentException( + "Unknown anchorEvent label '" + _anchorEventLabel + "'"); + _anchorEvent = idx; + } + + public Integer getAnchorEvent() { return _anchorEvent; } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public Integer getStudyId() + { + return _studyId; + } + + public void setStudyId(Integer studyId) + { + _studyId = studyId; + } + + public String getCohortName() + { + return _cohortName; + } + + public void setCohortName(String cohortName) + { + _cohortName = cohortName; + } + + public Integer getCohortId() + { + return _cohortId; + } + + public void setCohortId(Integer cohortId) + { + _cohortId = cohortId; + } + + public String getLabel() + { + return _label; + } + + public void setLabel(String label) + { + _label = label; + } + + public String getLabelShort() + { + return _labelShort; + } + + public void setLabelShort(String labelShort) + { + _labelShort = labelShort; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public Integer getRangeMin() + { + return _rangeMin; + } + + public void setRangeMin(Integer rangeMin) + { + _rangeMin = rangeMin; + } + + public Integer getRangeMax() + { + return _rangeMax; + } + + public void setRangeMax(Integer rangeMax) + { + _rangeMax = rangeMax; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } + + public Integer getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(Integer createdBy) + { + _createdBy = createdBy; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public Integer getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(Integer modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + } + + public static StudyDefinition fromJson(JSONObject json) + { + ObjectMapper mapper = new ObjectMapper(); + StudyDefinition sd = mapper.convertValue(json.toMap(), StudyDefinition.class); + + // In our JSON, Timepoints store the anchorEvent label, not an ID. Since the DB schema requires an int, we need + // to do that translation manually. Here, we store the anchorEvent by its index in the anchorEvent list. + Map idxByLabel = IntStream.range(0, sd.getAnchorEvents().size()) + .boxed() + .collect(Collectors.toMap( + i -> sd.getAnchorEvents().get(i).getLabel(), + i -> i)); + + sd.getTimepoints().forEach(tp -> tp.resolveAnchorEvent(idxByLabel)); + + return sd; + } + + public String toJson() throws JsonProcessingException + { + ObjectWriter ow = new ObjectMapper().writer(); + return ow.writeValueAsString(this); + } + + public static StudyDefinition getForId(int studyId) + { + // TODO: implement this. This should query the DB and return a populated StudyDefinition + + return null; + } +} diff --git a/Studies/resources/study/DemoStudy.json b/Studies/resources/study/DemoStudy.json new file mode 100644 index 000000000..07f419b06 --- /dev/null +++ b/Studies/resources/study/DemoStudy.json @@ -0,0 +1,43 @@ +{ + "studyName": "DemoStudy", + "label": "Demo Study", + "description": "This is a demo study", + "cohorts": [{ + "cohortName": "Group1", + "label": "Group 1", + "description": "This is the first group", + "isControlGroup": false, + "sortOrder": 1 + },{ + "cohortName": "Control", + "label": "Control Group", + "description": "This is the control group", + "isControlGroup": true, + "sortOrder": 2 + }], + "anchorEvents": [{ + "label": "Study Enrollment", + "description": "The represents Day 0 of the study", + "eventProviderName": "EnrollmentStart" + }], + "timepoints": [{ + "cohortId": null, + "label": "Day 0", + "labelShort": "D0", + "anchorEvent": "Study Enrollment" + },{ + "cohortName": "Group1", + "label": "Vaccination", + "labelShort": "V", + "anchorEvent": "Study Enrollment", + "rangeMin": 7, + "rangeMax": 10 + },{ + "cohortName": "Control", + "label": "Mock-Vaccination", + "labelShort": "V", + "anchorEvent": "Study Enrollment", + "rangeMin": 7, + "rangeMax": 10 + }] +} \ No newline at end of file diff --git a/Studies/src/org/labkey/studies/StudiesController.java b/Studies/src/org/labkey/studies/StudiesController.java index 58d6b51dd..8de8e05be 100644 --- a/Studies/src/org/labkey/studies/StudiesController.java +++ b/Studies/src/org/labkey/studies/StudiesController.java @@ -1,37 +1,16 @@ package org.labkey.studies; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.SimpleApiJsonForm; import org.labkey.api.action.SpringActionController; -import org.labkey.api.data.TableInfo; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DuplicateKeyException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.reader.TabLoader; -import org.labkey.api.resource.Resource; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.studies.StudiesService; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Path; -import org.labkey.api.util.URLHelper; +import org.labkey.api.studies.study.StudyDefinition; import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.HtmlView; import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.servlet.ModelAndView; -import java.io.IOException; -import java.sql.SQLException; -import java.util.List; import java.util.Map; public class StudiesController extends SpringActionController @@ -45,4 +24,26 @@ public StudiesController() { setActionResolver(_actionResolver); } + + + @RequiresPermission(AdminPermission.class) + public static class UpdateStudyDefinitionAction extends MutatingApiAction + { + @Override + public Object execute(SimpleApiJsonForm json, BindException errors) throws Exception + { + try + { + StudyDefinition sd = StudyDefinition.fromJson(json.getJsonObject()); + sd = StudiesManager.get().insertOrUpdateStudyDefinition(sd, getContainer(), getUser()); + + return new ApiSimpleResponse(Map.of("success", true, "studyDefinition", sd)); + } + catch (Exception e) + { + _log.error("Unable to import study definition", e); + return new ApiSimpleResponse("success", false); + } + } + } } diff --git a/Studies/src/org/labkey/studies/StudiesManager.java b/Studies/src/org/labkey/studies/StudiesManager.java index 469f52781..bf7926dcc 100644 --- a/Studies/src/org/labkey/studies/StudiesManager.java +++ b/Studies/src/org/labkey/studies/StudiesManager.java @@ -1,16 +1,416 @@ package org.labkey.studies; +import org.apache.commons.collections.map.CaseInsensitiveMap; +import org.apache.commons.io.IOUtils; +import org.json.JSONObject; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.discvrcore.test.AbstractIntegrationTest; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.SimpleUserSchema; +import org.labkey.api.query.UserSchema; +import org.labkey.api.resource.FileResource; +import org.labkey.api.resource.Resource; +import org.labkey.api.security.User; +import org.labkey.api.studies.study.StudyDefinition; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.TestContext; +import org.labkey.studies.query.StudiesUserSchema; + + +import java.io.FileInputStream; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.fasterxml.jackson.databind.type.LogicalType.Collection; + +@RunWith(Enclosed.class) public class StudiesManager { private static final StudiesManager _instance = new StudiesManager(); private StudiesManager() { - // prevent external construction with a private default constructor + } public static StudiesManager get() { return _instance; } + + public StudyDefinition insertOrUpdateStudyDefinition(StudyDefinition sd, Container c, User u) + { + StudiesSchema ss = StudiesSchema.getInstance(); + DbSchema schema = ss.getSchema(); + DbScope scope = schema.getScope(); + + UserSchema us = QueryService.get().getUserSchema(u, c, StudiesSchema.NAME); + + TableInfo tblStudies = us.getTable(StudiesSchema.TABLE_STUDIES); + TableInfo tblCohorts = us.getTable(StudiesSchema.TABLE_COHORTS); + TableInfo tblAnchorEvents = us.getTable(StudiesSchema.TABLE_ANCHOR_EVENTS); + TableInfo tblTimepoints = us.getTable(StudiesSchema.TABLE_EXPECTED_TIMEPOINTS); + + try (DbScope.Transaction tx = scope.ensureTransaction()) + { + sd.setContainer(c.getEntityId().toString()); + sd = upsertStudy(sd, tblStudies, c, u); + + upsertChildRecords( + sd.getRowId(), + sd.getCohorts(), + tblCohorts, + c, + u, + this::cohortToMap, + StudyDefinition.StudyCohort::getRowId, + StudyDefinition.StudyCohort::setRowId + ); + + upsertChildRecords( + sd.getRowId(), + sd.getAnchorEvents(), + tblAnchorEvents, + c, + u, + this::anchorToMap, + StudyDefinition.AnchorEvent::getRowId, + StudyDefinition.AnchorEvent::setRowId + ); + + upsertChildRecords( + sd.getRowId(), + sd.getTimepoints(), + tblTimepoints, + c, + u, + this::timepointToMap, + StudyDefinition.Timepoint::getRowId, + StudyDefinition.Timepoint::setRowId + ); + + tx.commit(); + } + catch (Exception x) + { + throw new RuntimeException("Failed to up‑sert StudyDefinition", x); + } + + return sd; + } + + private StudyDefinition upsertStudy(StudyDefinition sd, + TableInfo tbl, + Container c, + User u) throws Exception + { + Map map = studyToMap(sd); + BatchValidationException bve = new BatchValidationException(); + QueryUpdateService qus = tbl.getUpdateService(); + + List> rows = List.of(map); + List> ret; + + if (sd.getRowId() == null) + ret = qus.insertRows(u, c, rows, bve, null, null); + else + { + ret = qus.updateRows(u, c, rows, null, bve, null, null); + } + + if (bve.hasErrors()) + throw bve; + + sd.setRowId((Integer) ret.get(0).get("rowId")); + return sd; + } + + + private void upsertChildRecords(int studyRowId, + List incoming, + TableInfo tbl, + Container c, + User u, + Mapper mapper, + RowIdGetter getRowId, + RowIdSetter setRowId) throws Exception + { + QueryUpdateService qus = tbl.getUpdateService(); + + Set existing = new HashSet<>( + new TableSelector(tbl, PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), studyRowId), + null).getCollection(Integer.class) + ); + + List> inserts = new ArrayList<>(); + List insertBeans = new ArrayList<>(); + List> updates = new ArrayList<>(); + + for (T bean : incoming) + { + Map row = mapper.apply(bean); + row.put("studyId", studyRowId); + + Integer rk = getRowId.get(bean); + if (rk == null) + { + inserts.add(row); + insertBeans.add(bean); + } + else + { + updates.add(row); + existing.remove(rk); + } + } + + if (!existing.isEmpty()) + { + List> keys = existing.stream() + .map(rid -> Map.of("rowid", (Object) rid)) + .toList(); + + qus.deleteRows(u, c, keys, null, null); + } + + BatchValidationException bve = new BatchValidationException(); + + if (!inserts.isEmpty()) + { + List> ret = qus.insertRows(u, c, inserts, bve, null, null); + for (int i = 0; i < ret.size(); i++) + setRowId.set(insertBeans.get(i), (Integer) ret.get(i).get("rowId")); + } + + if (!updates.isEmpty()) + { + qus.updateRows(u, c, updates, null, bve, null, null); + } + + if (bve.hasErrors()) + throw bve; + } + + private Map studyToMap(StudyDefinition s) + { + Map m = new HashMap<>(); + if (s.getRowId() != null) + m.put("rowId", s.getRowId()); + m.put("name", s.getStudyName()); + m.put("label", s.getLabel()); + m.put("category", s.getCategory()); + m.put("description", s.getDescription()); + m.put("container", s.getContainer()); + return m; + } + + private Map cohortToMap(StudyDefinition.StudyCohort c) + { + Map m = new HashMap<>(); + if (c.getRowId() != null) + m.put("rowId", c.getRowId()); + m.put("cohortName", c.getCohortName()); + m.put("label", c.getLabel()); + m.put("category", c.getCategory()); + m.put("description", c.getDescription()); + m.put("isControlGroup",c.getIsControlGroup()); + m.put("sortOrder", c.getSortOrder()); + m.put("container", c.getContainer()); + return m; + } + + private Map anchorToMap(StudyDefinition.AnchorEvent a) + { + Map m = new HashMap<>(); + if (a.getRowId() != null) + m.put("rowId", a.getRowId()); + m.put("label", a.getLabel()); + m.put("description", a.getDescription()); + m.put("eventProviderName",a.getEventProviderName()); + m.put("container", a.getContainer()); + return m; + } + + private Map timepointToMap(StudyDefinition.Timepoint t) + { + Map m = new HashMap<>(); + if (t.getRowId() != null) + m.put("rowId", t.getRowId()); + m.put("cohortId", t.getCohortId()); + m.put("cohortName", t.getCohortName()); + m.put("label", t.getLabel()); + m.put("labelShort", t.getLabelShort()); + m.put("description", t.getDescription()); + m.put("anchorEvent", t.getAnchorEvent()); + m.put("rangeMin", t.getRangeMin()); + m.put("rangeMax", t.getRangeMax()); + m.put("container", t.getContainer()); + return m; + } + + @FunctionalInterface private interface Mapper { Map apply(T t); } + @FunctionalInterface private interface RowIdGetter { Integer get(T t); } + @FunctionalInterface private interface RowIdSetter { void set(T t, Integer id); } + + public static class TestCase extends AbstractIntegrationTest + { + public static final String PROJECT_NAME = "StudiesIntegrationTestFolder"; + + @BeforeClass + public static void setup() throws Exception + { + doInitialSetUp(PROJECT_NAME); + + Container project = ContainerManager.getForPath(PROJECT_NAME); + Set active = new HashSet<>(project.getActiveModules()); + active.add(ModuleLoader.getInstance().getModule(StudiesModule.NAME)); + project.setActiveModules(active); + } + + @AfterClass + public static void cleanup() + { + doCleanup(PROJECT_NAME); + } + + @Test + public void testStudyInsert() throws Exception + { + Container c = ContainerManager.getForPath(PROJECT_NAME); + Resource r = ModuleLoader.getInstance() + .getModule(StudiesModule.NAME) + .getModuleResource("study/DemoStudy.json"); + + if (!(r instanceof FileResource fr)) + throw new IllegalStateException("Expected a FileResource; got " + r); + + StudyDefinition sd; + try (InputStream is = new FileInputStream(fr.getFile())) + { + String jsonTxt = IOUtils.toString(is, StringUtilsLabKey.DEFAULT_CHARSET); + sd = StudyDefinition.fromJson(new JSONObject(jsonTxt)); + } + + sd = StudiesManager.get().insertOrUpdateStudyDefinition(sd, c, TestContext.get().getUser()); + + // 1. Verify insert + assertNotNull( "study rowId null after insert", sd.getRowId()); + sd.getCohorts().forEach(co -> assertNotNull("cohort rowId null", co.getRowId())); + sd.getAnchorEvents().forEach(ev -> assertNotNull("anchor rowId null", ev.getRowId())); + sd.getTimepoints().forEach(tp -> assertNotNull("timepoint rowId null", tp.getRowId())); + + StudiesSchema ss = StudiesSchema.getInstance(); + DbSchema schema = ss.getSchema(); + TableInfo tblStudies = schema.getTable(StudiesSchema.TABLE_STUDIES); + TableInfo tblCohorts = schema.getTable(StudiesSchema.TABLE_COHORTS); + TableInfo tblTP = schema.getTable(StudiesSchema.TABLE_EXPECTED_TIMEPOINTS); + + assertEquals(1, + new TableSelector(tblStudies, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("rowId"), sd.getRowId()), + null).getRowCount()); + + int cohortCount = sd.getCohorts().size(); + int timepointCount = sd.getTimepoints().size(); + + assertEquals(cohortCount, + new TableSelector(tblCohorts, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + assertEquals(timepointCount, + new TableSelector(tblTP, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + // 2. Update some values, add a cohort, delete a timepoint + sd.setLabel(sd.getLabel() + " (updated)"); + + StudyDefinition.StudyCohort firstCohort = sd.getCohorts().get(0); + firstCohort.setLabel(firstCohort.getLabel() + "-updated"); + + StudyDefinition.StudyCohort newCohort = new StudyDefinition.StudyCohort(); + newCohort.setCohortName("NEW"); + newCohort.setLabel("Brand-new cohort"); + sd.getCohorts().add(newCohort); + + StudyDefinition.Timepoint removedTp = sd.getTimepoints().remove(0); + + sd = StudiesManager.get().insertOrUpdateStudyDefinition(sd, c, TestContext.get().getUser()); + + assertNotNull("new cohort did not receive rowId", newCohort.getRowId()); + + assertEquals(cohortCount + 1, + new TableSelector(tblCohorts, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + assertEquals(timepointCount - 1, + new TableSelector(tblTP, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + String dbLabel = new TableSelector(tblStudies, + PageFlowUtil.set("label"), + new SimpleFilter(FieldKey.fromString("rowId"), sd.getRowId()), + null).getObject(String.class); + assertEquals(sd.getLabel(), dbLabel); + + String dbCohortLabel = new TableSelector(tblCohorts, + PageFlowUtil.set("label"), + new SimpleFilter(FieldKey.fromString("rowId"), firstCohort.getRowId()), + null).getObject(String.class); + assertEquals(firstCohort.getLabel(), dbCohortLabel); + + // 3. Delete the new cohort + sd.getCohorts().remove(newCohort); + sd = StudiesManager.get().insertOrUpdateStudyDefinition(sd, c, TestContext.get().getUser()); + + assertEquals(cohortCount, + new TableSelector(tblCohorts, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + // 4. Round-trip JSON export + JSONObject roundTrip = new JSONObject(sd.toJson()); + assertEquals(sd.getLabel(), roundTrip.getString("label")); + assertEquals(sd.getCohorts().size(), roundTrip.getJSONArray("cohorts").length()); + assertEquals(sd.getTimepoints().size(), roundTrip.getJSONArray("timepoints").length()); + } + } } \ No newline at end of file diff --git a/Studies/src/org/labkey/studies/StudiesModule.java b/Studies/src/org/labkey/studies/StudiesModule.java index f68d3746d..b4d204551 100644 --- a/Studies/src/org/labkey/studies/StudiesModule.java +++ b/Studies/src/org/labkey/studies/StudiesModule.java @@ -11,6 +11,8 @@ import org.labkey.api.query.QuerySchema; import org.labkey.api.security.roles.RoleManager; import org.labkey.api.studies.StudiesService; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.studies.query.StudiesUserSchema; import org.labkey.api.studies.security.StudiesDataAdminRole; import org.labkey.studies.query.StudiesUserSchema; import org.labkey.studies.study.StudiesFilterProvider; @@ -79,4 +81,12 @@ public QuerySchema createSchema(final DefaultSchema schema, Module module) } }); } + + @Override + public @NotNull Set getIntegrationTests() + { + return PageFlowUtil.set( + StudiesManager.TestCase.class + ); + } } \ No newline at end of file diff --git a/Studies/src/org/labkey/studies/query/LookupSetsTable.java b/Studies/src/org/labkey/studies/query/LookupSetsTable.java index 4d6ff8230..83c8c78ef 100644 --- a/Studies/src/org/labkey/studies/query/LookupSetsTable.java +++ b/Studies/src/org/labkey/studies/query/LookupSetsTable.java @@ -38,14 +38,14 @@ public UpdateService(SimpleUserSchema.SimpleTable ti) @Override protected void afterInsertUpdate(int count, BatchValidationException errors) { - LookupSetsManager.get().getCache().clear(); + StudiesUserSchema.repopulateCaches(getUserSchema().getUser(), getUserSchema().getContainer()); } @Override protected Map deleteRow(User user, Container container, Map oldRowMap) throws QueryUpdateServiceException, SQLException, InvalidKeyException { Map row = super.deleteRow(user, container, oldRowMap); - LookupSetsManager.get().getCache().clear(); + StudiesUserSchema.repopulateCaches(getUserSchema().getUser(), getUserSchema().getContainer()); return row; } @@ -53,7 +53,7 @@ protected Map deleteRow(User user, Container container, Map> getPropertySetNames() { Map> nameMap = (Map>) LookupSetsManager.get().getCache().get(LookupSetTable.getCacheKey(getTargetContainer())); @@ -81,6 +90,7 @@ private Map> getPropertySetNames() return nameMap; } + _log.debug("Populating lookup tables in StudiesUserSchema.getPropertySetNames() for container: " + getTargetContainer().getName()); nameMap = new CaseInsensitiveHashMap<>(); TableSelector ts = new TableSelector(_dbSchema.getTable(TABLE_LOOKUP_SETS), new SimpleFilter(FieldKey.fromString("container"), getTargetContainer().getId()), null); diff --git a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java index 4fc7a91d8..cd83a749c 100644 --- a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java +++ b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java @@ -291,4 +291,16 @@ public URLHelper getRedirectURL(Object o) throws Exception return DetailsURL.fromString("laboratory/setTableIncrementValue.view", getContainer()).getActionURL(); } } + + // This allows registration of this action without creating a dependency between laboratory and discvrcore + @UtilityAction(label = "Manage File Roots", description = "This standalone file root management action can be used on folder types that do not support the normal 'Manage Folder' UI.") + @RequiresPermission(AdminPermission.class) + public class ManageFileRootAction extends SimpleRedirectAction + { + @Override + public URLHelper getRedirectURL(Object o) throws Exception + { + return DetailsURL.fromString("admin/manageFileRoot.view", getContainer()).getActionURL(); + } + } } diff --git a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java index a99627487..152241120 100644 --- a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java +++ b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java @@ -24,6 +24,9 @@ import org.labkey.api.query.DetailsURL; import org.labkey.api.settings.AdminConsole; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.DeveloperMenuNavTrees; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.PopupDeveloperView; import org.labkey.api.view.WebPartFactory; import java.util.Collection; @@ -70,6 +73,9 @@ protected void init() public void doStartup(ModuleContext moduleContext) { AdminConsole.addLink(AdminConsole.SettingsLinkType.Management, "site utility actions", DetailsURL.fromString("discvrcore/showUtilityActions.view", ContainerManager.getRoot()).getActionURL()); + PopupDeveloperView.registerMenuProvider((c, user, trees) -> { + trees.add(DeveloperMenuNavTrees.Section.tools, new NavTree("Show Utility Actions", DetailsURL.fromString("discvrcore/showUtilityActions.view", c).getActionURL())); + }); } @Override diff --git a/jbrowse/package-lock.json b/jbrowse/package-lock.json index a59c3c97c..b2b8218e0 100644 --- a/jbrowse/package-lock.json +++ b/jbrowse/package-lock.json @@ -3091,9 +3091,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.41.2", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.41.2.tgz", - "integrity": "sha512-ninfc/+Sj5+8Zla9bY2j/4fSy41OS27YAHKtDFPnu52QkC8WsOYh3JFI5PkU6Rn+xIp0In4P6d5Qn/yluJRC/w==" + "version": "1.42.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.42.0.tgz", + "integrity": "sha512-fh65cogl+8HNe9+3OHzI6ygkaa+ARJbKyUopguy3NqtOWIAXAA21GNxayRoeVf9m1n2Kpubqk4QS2z/+WCzf8A==" }, "node_modules/@labkey/build": { "version": "8.5.0", @@ -3133,12 +3133,12 @@ } }, "node_modules/@labkey/components": { - "version": "6.50.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.50.1.tgz", - "integrity": "sha512-g6DDCg3rsoCzkJDTRAtUFdGskyufFipJB50mmgQhEgtyEEbaXB6rF/Ny8ytPCZOmpA5yL2+pEtqFoXlvyXBMDg==", + "version": "6.52.5", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.52.5.tgz", + "integrity": "sha512-hkesy0zyVEpADMfK4CQN9ZPWRjGFGfqS7h1QgRazp0QiHgXI3i6Va39lMSBdZb27bDeQd5V78Xze4tclHO/xIQ==", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.41.2", + "@labkey/api": "1.42.0", "@testing-library/dom": "~10.4.0", "@testing-library/jest-dom": "~6.6.3", "@testing-library/react": "~16.3.0", @@ -5361,14 +5361,41 @@ } }, "node_modules/call-bind": { - "version": "1.0.7", - "license": "MIT", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", "dependencies": { + "call-bind-apply-helpers": "^1.0.0", "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.1" + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" }, "engines": { "node": ">= 0.4" @@ -6495,6 +6522,19 @@ "tslib": "^2.0.3" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/duplexer": { "version": "0.1.2", "dev": true, @@ -6619,11 +6659,9 @@ } }, "node_modules/es-define-property": { - "version": "1.0.0", - "license": "MIT", - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "engines": { "node": ">= 0.4" } @@ -6639,6 +6677,17 @@ "version": "1.5.4", "license": "MIT" }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -6949,10 +6998,17 @@ } }, "node_modules/for-each": { - "version": "0.3.3", - "license": "MIT", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", "dependencies": { - "is-callable": "^1.1.3" + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/foreground-child": { @@ -7252,14 +7308,20 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "license": "MIT", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -7268,6 +7330,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-value": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/get-value/-/get-value-3.0.1.tgz", @@ -7352,10 +7426,11 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "license": "MIT", - "dependencies": { - "get-intrinsic": "^1.1.3" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -7403,19 +7478,10 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-proto": { - "version": "1.0.3", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { - "version": "1.0.3", - "license": "MIT", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", "engines": { "node": ">= 0.4" }, @@ -7887,7 +7953,8 @@ }, "node_modules/is-callable": { "version": "1.2.7", - "license": "MIT", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", "engines": { "node": ">= 0.4" }, @@ -8076,10 +8143,11 @@ } }, "node_modules/is-typed-array": { - "version": "1.1.13", - "license": "MIT", + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", "dependencies": { - "which-typed-array": "^1.1.14" + "which-typed-array": "^1.1.16" }, "engines": { "node": ">= 0.4" @@ -8411,6 +8479,14 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/md5": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", @@ -9404,20 +9480,49 @@ } }, "node_modules/pbkdf2": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", - "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.3.tgz", + "integrity": "sha512-wfRLBZ0feWRhCIkoMB6ete7czJcnNnqRpcoWQBLqatqXXmelSRqfdDK4F3u9T2s2cXas/hQJcryI/4lAL+XTlA==", "dependencies": { - "create-hash": "^1.1.2", - "create-hmac": "^1.1.4", - "ripemd160": "^2.0.1", - "safe-buffer": "^5.0.1", - "sha.js": "^2.4.8" + "create-hash": "~1.1.3", + "create-hmac": "^1.1.7", + "ripemd160": "=2.0.1", + "safe-buffer": "^5.2.1", + "sha.js": "^2.4.11", + "to-buffer": "^1.2.0" }, "engines": { "node": ">=0.12" } }, + "node_modules/pbkdf2/node_modules/create-hash": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.1.3.tgz", + "integrity": "sha512-snRpch/kwQhcdlnZKYanNF1m0RDlrCdSKQaH87w1FCFPVPNCQ/Il9QJKAX2jVBZddRdaHBMC+zXa9Gw9tmkNUA==", + "dependencies": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "sha.js": "^2.4.0" + } + }, + "node_modules/pbkdf2/node_modules/hash-base": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-2.0.2.tgz", + "integrity": "sha512-0TROgQ1/SxE6KmxWSvXHvRj90/Xo1JvZShofnYF+f6ZsGtR4eES7WfrQzPalmyagfKZCXpVnitiRebZulWsbiw==", + "dependencies": { + "inherits": "^2.0.1" + } + }, + "node_modules/pbkdf2/node_modules/ripemd160": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.1.tgz", + "integrity": "sha512-J7f4wutN8mdbV08MJnXibYpCOPHR+yzy+iQ/AsjMv2j8cLavQ8VGagDFUwwTAdF8FmRKVeNpbTTEwNHCW1g94w==", + "dependencies": { + "hash-base": "^2.0.0", + "inherits": "^2.0.1" + } + }, "node_modules/performance-now": { "version": "2.1.0", "license": "MIT", @@ -11555,6 +11660,24 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/to-buffer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.1.tgz", + "integrity": "sha512-tB82LpAIWjhLYbqjx3X4zEeHN6M8CiuOEy2JY8SEQVdYRe3CCHOFaqrBW1doLDrfpWhplcW7BL+bO3/6S3pcDQ==", + "dependencies": { + "isarray": "^2.0.5", + "safe-buffer": "^5.2.1", + "typed-array-buffer": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/to-buffer/node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -11676,6 +11799,19 @@ "node": ">= 0.6" } }, + "node_modules/typed-array-buffer": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", + "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", + "dependencies": { + "call-bound": "^1.0.3", + "es-errors": "^1.3.0", + "is-typed-array": "^1.1.14" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/typescript": { "version": "5.8.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", @@ -12328,13 +12464,16 @@ } }, "node_modules/which-typed-array": { - "version": "1.1.15", - "license": "MIT", + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", + "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==", "dependencies": { "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", - "for-each": "^0.3.3", - "gopd": "^1.0.1", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", "has-tostringtag": "^1.0.2" }, "engines": { diff --git a/jbrowse/resources/views/begin.html b/jbrowse/resources/views/begin.html index d42ddad7b..50c216bb6 100644 --- a/jbrowse/resources/views/begin.html +++ b/jbrowse/resources/views/begin.html @@ -1,19 +1,9 @@ - - The JBrowse module is part of DISCVR-Seq. It provides a wrapper around the JBrowse Genome Browser, which lets users rapidly take data generated or uploaded into DISCRV-Seq and view it using JBrowse. -The module ships with a version of JBrowse, meaning very little extra configuration is required in order to start using these tools. The primary reasons we created this wrapper around JBrowse are: +The module ships with a version of JBrowse 2, meaning very little extra configuration is required in order to start using these tools. The primary reasons we created this wrapper around JBrowse are:

  • We selected JBrowse as the browser because it is a modern web-based genome viewer, with a large support base
  • This wrapper makes is very simple for non-technical users to take data uploaded into DISCVR-Seq and use the power of JBrowse to see your data. While we find JBrowse easy to use, preparing samples for the browser often requires command-line tools or was otherwise beyond the level of a typical scientist
  • For our usage, we find it important to be able to quickly view different combinations of files. For example, after running an alignment or calling SNPs on a set of samples, you might want to view these specific samples in conjunction with a specific set of genome annotations (coding regions, regulatory regions, site of known variation, etc).
    • -
  • The files createde by genome-scale projects can become large. If we allow users to create many different JBrowse databases, we need to be careful about the amount of files this creates. The JBrowse module does a fair amount of work behind the scenes to avoid duplicating files. For example, if the same genome is used as the base for 100s of JBrowse sessions, there will only be one copy of the processed sequences files saved. The same is true for all tracks that are loaded.
    • +
  • The files created by genome-scale projects can become large. If we allow users to create many different JBrowse databases, we need to be careful about the amount of files this creates. The JBrowse module does a fair amount of work behind the scenes to avoid duplicating files. For example, if the same genome is used as the base for 100s of JBrowse sessions, there will only be one copy of the processed sequences files saved. The same is true for all tracks that are loaded.
\ No newline at end of file diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseManager.java b/jbrowse/src/org/labkey/jbrowse/JBrowseManager.java index ec0f92945..c1039d844 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseManager.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseManager.java @@ -236,7 +236,19 @@ public static class TestCase extends Assert public void testJBrowseCli() throws Exception { File exe = JBrowseManager.get().getJbrowseCli(); - String output = new SimpleScriptWrapper(_log).executeWithOutput(Arrays.asList(exe.getPath(), "help")); + SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); + wrapper.setThrowNonZeroExits(false); + + String output = wrapper.executeWithOutput(Arrays.asList(exe.getPath(), "help")); + if (wrapper.getLastReturnCode() != 0) + { + _log.error("Non-zero exit from testJBrowseCli: " + wrapper.getLastReturnCode()); + wrapper.getCommandsExecuted().forEach(_log::error); + _log.error("output: "); + _log.error(output); + + throw new RuntimeException("Non-zero exit running testJBrowseCli: " + wrapper.getLastReturnCode()); + } assertTrue("Malformed output", output.contains("Add an assembly to a JBrowse 2 configuration")); } diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java index a8fbe8888..9c118d2ee 100644 --- a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java +++ b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java @@ -17,6 +17,7 @@ import au.com.bytecode.opencsv.CSVReader; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; import org.json.JSONException; @@ -25,7 +26,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.CommandResponse; import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.SimplePostCommand; import org.labkey.remoteapi.query.Filter; import org.labkey.remoteapi.query.InsertRowsCommand; import org.labkey.remoteapi.query.SelectRowsCommand; @@ -53,6 +56,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.Date; import java.util.List; import java.util.Map; @@ -542,6 +546,22 @@ private void createGenomeFeatures(int genomeId) throws IOException, CommandExcep ic.execute(cn, getProjectName()); } + private JSONArray getSearchResults(Map queryParams) throws CommandException, IOException + { + Connection connection = createDefaultConnection(); + + Date start = new Date(); + SimplePostCommand command = new SimplePostCommand("jbrowse", "luceneQuery"); + command.setParameters(queryParams); + command.setTimeout(WAIT_FOR_PAGE * 3); + CommandResponse response = command.execute(connection, getProjectName()); + log("JBrowse search time: " + DurationFormatUtils.formatDurationWords(new Date().getTime() - start.getTime(), true, true)); + log(response.getText()); + + // NOTE: the response is ndjson. This converts it into a more standard JSONArray form: + return new JSONArray("[" + StringUtils.join(response.getText().split("\n"), ",") + "]"); + } + @Override public List getAssociatedModules() { @@ -573,24 +593,12 @@ private void testFullTextSearch() throws Exception // all // this should return 143 results. We can't make any other assumptions about the content - String url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=143"; - beginAt(url, WAIT_FOR_PAGE * 2); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - String jsonString = getText(Locator.tagWithClass("pre", "data")); - JSONObject mainJsonObject = new JSONObject(jsonString); - JSONArray jsonArray = mainJsonObject.getJSONArray("data"); + JSONArray jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 143)); Assert.assertEquals(143, jsonArray.length()); // stringType: // ref equals A - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3AA"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "ref:A")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -598,13 +606,7 @@ private void testFullTextSearch() throws Exception } // alt does not equal C - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-alt%3AC"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -alt:C")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -612,13 +614,7 @@ private void testFullTextSearch() throws Exception } // ref contains A - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3A*A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "ref:*A*")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -626,13 +622,7 @@ private void testFullTextSearch() throws Exception } // alt does not contain AA - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-alt%3A*AA*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -alt:*AA*")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -640,13 +630,7 @@ private void testFullTextSearch() throws Exception } // IMPACT starts with HI - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=IMPACT%3AHI*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "IMPACT:HI*")); Assert.assertEquals(1, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -654,13 +638,7 @@ private void testFullTextSearch() throws Exception } // ref ends with TA - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3A*TA"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "ref:*TA")); Assert.assertEquals(5, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -668,13 +646,7 @@ private void testFullTextSearch() throws Exception } // IMPACT is empty - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-IMPACT%3A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -IMPACT:*")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -682,13 +654,7 @@ private void testFullTextSearch() throws Exception } // IMPACT is not empty - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=IMPACT%3A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "IMPACT:*")); Assert.assertEquals(3, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -696,23 +662,11 @@ private void testFullTextSearch() throws Exception } // variableSamplesType in set TestGroup - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=variableSamples%3A~!TestGroup!~"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "variableSamples:~!TestGroup!~")); Assert.assertEquals(100, jsonArray.length()); // variable in m00004 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=variableSamples%3Am00004"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "variableSamples:m00004")); Assert.assertEquals(79, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -731,13 +685,7 @@ private void testFullTextSearch() throws Exception } // not variable in m00004 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-variableSamples%3Am00004"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -variableSamples:m00004")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -767,13 +715,7 @@ private void testFullTextSearch() throws Exception } // variable in all of m00004, m00013, m00029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=%252BvariableSamples%3Am00004%20%252BvariableSamples%3Am00013%20%252BvariableSamples%3Am00029"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "%2BvariableSamples:m00004 %2BvariableSamples:m00013 %2BvariableSamples:m00029")); Assert.assertEquals(69, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -801,13 +743,7 @@ private void testFullTextSearch() throws Exception } // variable in any of m00004, m00013, m00029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=variableSamples%3Am00004%20OR%20variableSamples%3Am00013%20OR%20variableSamples%3Am00029"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "variableSamples:m00004 OR variableSamples:m00013 OR variableSamples:m00029")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -833,13 +769,7 @@ private void testFullTextSearch() throws Exception } // not variable in any of m00004, m00013, m00029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-variableSamples%3Am00004%20AND%20*%3A*%20-variableSamples%3Am00013%20AND%20*%3A*%20-variableSamples%3Am00029"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -variableSamples:m00004 AND *:* -variableSamples:m00013 AND *:* -variableSamples:m00029")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -885,13 +815,7 @@ private void testFullTextSearch() throws Exception } // not variable in one of m00004, m00013, m00029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-variableSamples%3Am00004%20OR%20*%3A*%20-variableSamples%3Am00013%20OR%20*%3A*%20-variableSamples%3Am00029"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -variableSamples:m00004 OR *:* -variableSamples:m00013 OR *:* -variableSamples:m00029")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -935,13 +859,7 @@ private void testFullTextSearch() throws Exception } // is empty - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-variableSamples%3A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -variableSamples:*")); Assert.assertEquals(5, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -949,13 +867,7 @@ private void testFullTextSearch() throws Exception } // is not empty - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=variableSamples%3A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "variableSamples:*")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -965,13 +877,7 @@ private void testFullTextSearch() throws Exception // numericType, int and float: // AC = 12 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AC%3A%5B12%20TO%2012%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AC:[12 TO 12]")); Assert.assertEquals(3, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -979,13 +885,7 @@ private void testFullTextSearch() throws Exception } // AC != 88 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AC%3A%5B*%20TO%2088%7D%20OR%20AC%3A%7B88%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AC:[* TO 88} OR AC:{88 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -993,13 +893,7 @@ private void testFullTextSearch() throws Exception } // AC > 88 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AC%3A%7B88%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AC:{88 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1007,13 +901,7 @@ private void testFullTextSearch() throws Exception } // AC >= 88 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AC%3A%5B88%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AC:[88 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1021,13 +909,7 @@ private void testFullTextSearch() throws Exception } // start < 137 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=start%3A%5B*%20TO%20137%7D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "start:[* TO 137}")); Assert.assertEquals(2, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1035,13 +917,7 @@ private void testFullTextSearch() throws Exception } // end <= 440 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=end%3A%5B*%20TO%20440%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "end:[* TO 440]")); Assert.assertEquals(7, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1049,13 +925,7 @@ private void testFullTextSearch() throws Exception } // AF = 0.532 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B0.531999%20TO%200.5320010000000001%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[0.531999 TO 0.5320010000000001]")); Assert.assertEquals(1, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1063,13 +933,7 @@ private void testFullTextSearch() throws Exception } // AF != 0.029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B*%20TO%200.028999%5D%20OR%20AF%3A%5B0.029001000000000002%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[* TO 0.028999] OR AF:[0.029001000000000002 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1077,13 +941,7 @@ private void testFullTextSearch() throws Exception } // AF > 0.532 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B0.5320010000000001%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[0.5320010000000001 TO *]")); Assert.assertEquals(18, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1091,13 +949,7 @@ private void testFullTextSearch() throws Exception } // AF >= 0.029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B0.029%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[0.029 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1105,13 +957,7 @@ private void testFullTextSearch() throws Exception } // AF < 0.029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B*%20TO%200.028999%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[* TO 0.028999]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1119,13 +965,7 @@ private void testFullTextSearch() throws Exception } // AF <= 0.029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B*%20TO%200.029%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[* TO 0.029]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1137,13 +977,7 @@ private void testFullTextSearch() throws Exception // contig := 1 // ref := A // should be 100 results and each should be ref = A - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=contig%3A%3D1%26ref%3A%3DA&pageSize=200"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "contig:=1&ref:=A", "pageSize", 200)); Assert.assertEquals(104, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1152,14 +986,7 @@ private void testFullTextSearch() throws Exception } // Default genomic position sort (ascending) - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100)); long previousGenomicPosition = Long.MIN_VALUE; for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1169,14 +996,7 @@ private void testFullTextSearch() throws Exception } // Sort by alt, ascending - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100, "sortField", "alt")); String previousAlt = ""; for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1186,14 +1006,7 @@ private void testFullTextSearch() throws Exception } // Sort by alt, descending - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt&sortReverse=true"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100, "sortField", "alt", "sortReverse", "true")); previousAlt = "ZZZZ"; // Assuming 'Z' is higher than any character in your data for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1203,14 +1016,7 @@ private void testFullTextSearch() throws Exception } // Sort by af, ascending - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100, "sortField", "AF")); double previousAf = -1.0; for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1220,14 +1026,7 @@ private void testFullTextSearch() throws Exception } // Sort by af, descending - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF&sortReverse=true"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100, "sortField", "AF", "sortReverse", "true")); previousAf = 2.0; // Assuming 'af' is <= 1.0 for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); diff --git a/singlecell/resources/chunks/FindClustersAndDimRedux.R b/singlecell/resources/chunks/FindClustersAndDimRedux.R index b7fdc7c0f..47ecfa6c2 100644 --- a/singlecell/resources/chunks/FindClustersAndDimRedux.R +++ b/singlecell/resources/chunks/FindClustersAndDimRedux.R @@ -15,24 +15,26 @@ if (!reticulate::py_module_available(module = 'leidenalg')) { } } +if (all(is.null(clusterResolutions)) || clusterResolutions == '') { + clusterResolutions <- c(0.2, 0.4, 0.6, 0.8, 1.2) +} else if (is.character(clusterResolutions)) { + clusterResolutionsOrig <- clusterResolutions + clusterResolutions <- gsub(clusterResolutions, pattern = ' ', replacement = '') + clusterResolutions <- unlist(strsplit(clusterResolutions, split = ',')) + clusterResolutions <- as.numeric(clusterResolutions) + if (any(is.na(clusterResolutions))) { + stop(paste0('Some values for clusterResolutions were not numeric: ', clusterResolutionsOrig)) + } +} else if (is.numeric(clusterResolutions)) { + # No action needed +} else { + stop('Must provide a value for clusterResolutions') +} + for (datasetId in names(seuratObjects)) { printName(datasetId) seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) - if (all(is.null(clusterResolutions))) { - clusterResolutions <- c(0.2, 0.4, 0.6, 0.8, 1.2) - } else if (is.character(clusterResolutions)) { - clusterResolutionsOrig <- clusterResolutions - clusterResolutions <- gsub(clusterResolutions, pattern = ' ', replacement = '') - clusterResolutions <- unlist(strsplit(clusterResolutions, split = ',')) - clusterResolutions <- as.numeric(clusterResolutions) - if (any(is.na(clusterResolutions))) { - stop(paste0('Some values for clusterResolutions were not numeric: ', clusterResolutionsOrig)) - } - } else { - stop('Must provide a value for clusterResolutions') - } - seuratObj <- CellMembrane::FindClustersAndDimRedux(seuratObj, minDimsToUse = minDimsToUse, useLeiden = useLeiden, clusterResolutions = clusterResolutions) saveData(seuratObj, datasetId) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java index ec56a127c..7a646eb38 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java @@ -33,7 +33,8 @@ public Provider() put("height", 150); put("width", 600); put("delimiter", DELIM); - }}, null) + }}, null), + ToolParameterDescriptor.create("useDplyr", "Use dplyr", "If checked, the subset will be executed using dplyr::filter rather than Seurat::subset. This should allow more complex expressions to be used, including negations", "checkbox", null, false) ), List.of("/sequenceanalysis/field/TrimmingTextArea.js"), null); } @@ -70,6 +71,9 @@ protected List loadChunkFromFile() throws PipelineJobException final String[] values = val.split(DELIM); + ToolParameterDescriptor pd2 = getProvider().getParameterByName("useDplyr"); + final boolean useDplyr = pd2.extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); + List ret = new ArrayList<>(); for (String line : super.loadChunkFromFile()) { @@ -81,15 +85,23 @@ protected List loadChunkFromFile() throws PipelineJobException ret.add("\tif (!is.null(seuratObj)) {"); ret.add("\tprint(paste0('Subsetting dataset: ', datasetId, ' with the expression: " + subsetEscaped + "'))"); - ret.add("\t\tcells <- c()"); - ret.add("\t\ttryCatch({"); - ret.add("\t\t\tcells <- WhichCells(seuratObj, expression = " + subset + ")"); - ret.add("\t\t}, error = function(e){"); - ret.add("\t\t\tif (!is.null(e) && e$message == 'Cannot find cells provided') {"); - ret.add("\t\t\t\tprint(paste0('There were no cells remaining after the subset: ', '" + subsetEscaped + "'))"); - ret.add("\t\t\t}"); - ret.add("\t\t})"); - ret.add(""); + if (useDplyr) + { + ret.add("\t\t\tcells <- rownames(seuratObj@meta.data |> dplyr::filter( " + subset + " ))"); + } + else + { + ret.add("\t\tcells <- c()"); + ret.add("\t\ttryCatch({"); + ret.add("\t\t\tcells <- WhichCells(seuratObj, expression = " + subset + ")"); + ret.add("\t\t}, error = function(e){"); + ret.add("\t\t\tif (!is.null(e) && e$message == 'Cannot find cells provided') {"); + ret.add("\t\t\t\tprint(paste0('There were no cells remaining after the subset: ', '" + subsetEscaped + "'))"); + ret.add("\t\t\t}"); + ret.add("\t\t})"); + ret.add(""); + } + ret.add("\t\tif (length(cells) == 0) {"); ret.add("\t\t\tprint(paste0('There were no cells after subsetting for dataset: ', datasetId, ', with subset: ', '" + subsetEscaped + "'))"); ret.add("\t\t\tseuratObj <- NULL"); diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index eb7ae055d..d4ddc7326 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -1,5 +1,6 @@ package org.labkey.singlecell.pipeline.singlecell; +import au.com.bytecode.opencsv.CSVReader; import htsjdk.samtools.util.IOUtil; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -9,9 +10,11 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.reader.Readers; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.BcftoolsRunner; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; @@ -40,8 +43,8 @@ public class VireoHandler extends AbstractParameterizedOutputHandler(PageFlowUtil.set("sequenceanalysis/field/SequenceOutputFileSelectorField.js")), Arrays.asList( - ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex", "ldk-integerfield", new JSONObject(){{ - put("allowBlank", false); + ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex. This can be blank only if a reference VCF is provided.", "ldk-integerfield", new JSONObject(){{ + }}, null), ToolParameterDescriptor.create("maxDepth", "Max Depth", "At a position, read maximally INT reads per input file, to avoid excessive memory usage", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); @@ -111,6 +114,13 @@ public void init(JobContext ctx, List inputFiles, List inputFiles, JobContext c File barcodesGz = getBarcodesFile(inputFiles.get(0).getFile()); File bam = getBamFile(inputFiles.get(0).getFile()); - File barcodes = new File(ctx.getWorkingDirectory(), "barcodes.csv"); + File barcodes = new File(ctx.getWorkingDirectory(), "barcodes.tsv"); try (BufferedReader reader = IOUtil.openFileForBufferedUtf8Reading(barcodesGz); PrintWriter writer = PrintWriters.getPrintWriter(barcodes)) { String line; @@ -201,12 +211,6 @@ public void processFilesRemote(List inputFiles, JobContext c cellsnp.add(maxThreads.toString()); } - cellsnp.add("--minMAF"); - cellsnp.add("0.1"); - - cellsnp.add("--minCOUNT"); - cellsnp.add("100"); - String maxDepth = StringUtils.trimToNull(ctx.getParams().optString("maxDepth")); if (maxDepth != null) { @@ -239,6 +243,12 @@ public void processFilesRemote(List inputFiles, JobContext c cellsnp.add("--chrom"); cellsnp.add(contigs); } + + cellsnp.add("--minMAF"); + cellsnp.add("0.1"); + + cellsnp.add("--minCOUNT"); + cellsnp.add("100"); } new SimpleScriptWrapper(ctx.getLogger()).execute(cellsnp); @@ -256,6 +266,46 @@ public void processFilesRemote(List inputFiles, JobContext c } } + File cellSnpBaseVcf = new File(cellsnpDir, "cellSNP.base.vcf.gz"); + if (!cellSnpBaseVcf.exists()) + { + throw new PipelineJobException("Unable to find cellsnp base VCF"); + } + + + File cellSnpCellsVcf = new File(cellsnpDir, "cellSNP.cells.vcf.gz"); + if (!cellSnpCellsVcf.exists()) + { + throw new PipelineJobException("Unable to find cellsnp calls VCF"); + } + + int vcfFile = ctx.getParams().optInt(REF_VCF, -1); + File refVcfSubset = null; + if (vcfFile > -1) + { + File vcf = ctx.getSequenceSupport().getCachedData(vcfFile); + if (vcf == null || !vcf.exists()) + { + throw new PipelineJobException("Unable to find file with ID: " + vcfFile); + } + + refVcfSubset = new File(ctx.getWorkingDirectory(), vcf.getName()); + BcftoolsRunner bcftoolsRunner = new BcftoolsRunner(ctx.getLogger()); + bcftoolsRunner.execute(Arrays.asList( + BcftoolsRunner.getBcfToolsPath().getAbsolutePath(), + "view", + vcf.getPath(), + "-R", + cellSnpCellsVcf.getPath(), + "-Oz", + "-o", + refVcfSubset.getPath() + )); + + ctx.getFileManager().addIntermediateFile(refVcfSubset); + ctx.getFileManager().addIntermediateFile(new File(refVcfSubset.getPath() + ".tbi")); + } + List vireo = new ArrayList<>(); vireo.add("vireo"); vireo.add("-c"); @@ -270,15 +320,20 @@ public void processFilesRemote(List inputFiles, JobContext c vireo.add("-o"); vireo.add(ctx.getWorkingDirectory().getPath()); - int nDonors = ctx.getParams().optInt("nDonors", 0); boolean storeCellSnpVcf = ctx.getParams().optBoolean("storeCellSnpVcf", false); - if (nDonors == 0) + if (refVcfSubset != null) { - throw new PipelineJobException("Must provide nDonors"); + vireo.add("-d"); + vireo.add(refVcfSubset.getPath()); } - vireo.add("-N"); - vireo.add(String.valueOf(nDonors)); + // Note: this value should be checked in init(). It is required only if refVCF is null + int nDonors = ctx.getParams().optInt("nDonors", -1); + if (nDonors > -1) + { + vireo.add("-N"); + vireo.add(String.valueOf(nDonors)); + } if (nDonors == 1) { @@ -312,54 +367,81 @@ else if (outFiles.length > 1) so.setName(inputFiles.get(0).getName() + ": Vireo Demultiplexing"); } so.setCategory("Vireo Demultiplexing"); - ctx.addSequenceOutput(so); - } + StringBuilder description = new StringBuilder(); + if (vcfFile > -1) + { + description.append("Reference VCF ID: \n").append(vcfFile); + } - File cellSnpBaseVcf = new File(cellsnpDir, "cellSNP.base.vcf.gz"); - if (!cellSnpBaseVcf.exists()) - { - throw new PipelineJobException("Unable to find cellsnp base VCF"); - } + File summary = new File(ctx.getOutputDir(), "summary.tsv"); + if (!summary.exists()) + { + throw new PipelineJobException("Missing file: " + summary.getPath()); + } + description.append("Results:\n"); + try (CSVReader reader = new CSVReader(Readers.getReader(summary), '\t')) + { + String[] line; + while ((line = reader.readNext()) != null) + { + if ("Var1".equals(line[0])) + { + continue; + } - File cellSnpCellsVcf = new File(cellsnpDir, "cellSNP.cells.vcf.gz"); - if (!cellSnpCellsVcf.exists()) - { - throw new PipelineJobException("Unable to find cellsnp calls VCF"); - } + description.append(line[0]).append(": ").append(line[1]).append("\n"); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } - sortAndFixVcf(cellSnpBaseVcf, genome, ctx.getLogger()); - sortAndFixVcf(cellSnpCellsVcf, genome, ctx.getLogger()); + so.setDescription(StringUtils.trimToEmpty(description.toString())); + ctx.addSequenceOutput(so); + } if (storeCellSnpVcf) { + File fixedVcf = sortAndFixVcf(cellSnpBaseVcf, genome, ctx.getLogger(), ctx.getWorkingDirectory()); + SequenceOutputFile so = new SequenceOutputFile(); so.setReadset(inputFiles.get(0).getReadset()); so.setLibrary_id(inputFiles.get(0).getLibrary_id()); - so.setFile(cellSnpCellsVcf); + so.setFile(fixedVcf); if (so.getReadset() != null) { so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Cellsnp-lite VCF"); } else { - so.setName(inputFiles.get(0).getName() + ": Cellsnp-lite VCF"); + so.setName(inputFiles.get(0).getName() + ": Cellsnp-lite Base VCF"); } so.setCategory("VCF File"); ctx.addSequenceOutput(so); } + else + { + ctx.getFileManager().addIntermediateFile(cellSnpBaseVcf.getParentFile()); + } } - private void sortAndFixVcf(File vcf, ReferenceGenome genome, Logger log) throws PipelineJobException + private File sortAndFixVcf(File vcf, ReferenceGenome genome, Logger log, File outDir) throws PipelineJobException { + File outVcf = new File(outDir, vcf.getName()); + // This original VCF is generally not properly sorted, and has an invalid index. This is redundant, the VCF is not that large: try { - SequencePipelineService.get().sortROD(vcf, log, 2); - SequenceAnalysisService.get().ensureVcfIndex(vcf, log, true); + FileUtils.copyFile(vcf, outVcf); + SequencePipelineService.get().sortROD(outVcf, log, 2); + SequenceAnalysisService.get().ensureVcfIndex(outVcf, log, true); + + new UpdateVCFSequenceDictionary(log).execute(outVcf, genome.getSequenceDictionary()); + SequenceAnalysisService.get().ensureVcfIndex(outVcf, log); - new UpdateVCFSequenceDictionary(log).execute(vcf, genome.getSequenceDictionary()); - SequenceAnalysisService.get().ensureVcfIndex(vcf, log); + return outVcf; } catch (IOException e) {