From f21bfec4ffdd5e1fa3a4ae309dbc57f59a5222db Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 1 Mar 2025 08:52:23 -0800 Subject: [PATCH 1/9] Prefer bcftools/liftover --- .../api/sequenceanalysis/run}/LiftoverBcfToolsWrapper.java | 3 +-- .../org/labkey/sequenceanalysis/analysis/LiftoverHandler.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) rename SequenceAnalysis/{src/org/labkey/sequenceanalysis/run/util => api-src/org/labkey/api/sequenceanalysis/run}/LiftoverBcfToolsWrapper.java (96%) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/LiftoverBcfToolsWrapper.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/LiftoverBcfToolsWrapper.java similarity index 96% rename from SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/LiftoverBcfToolsWrapper.java rename to SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/LiftoverBcfToolsWrapper.java index 1a7c575ca..e3672883e 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/LiftoverBcfToolsWrapper.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/LiftoverBcfToolsWrapper.java @@ -1,11 +1,10 @@ -package org.labkey.sequenceanalysis.run.util; +package org.labkey.api.sequenceanalysis.run; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.api.sequenceanalysis.run.PicardWrapper; import java.io.File; import java.io.IOException; diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java index 934a0adea..93189cde9 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java @@ -37,7 +37,7 @@ import org.labkey.api.writer.PrintWriters; import org.labkey.sequenceanalysis.SequenceAnalysisModule; import org.labkey.sequenceanalysis.pipeline.ProcessVariantsHandler; -import org.labkey.sequenceanalysis.run.util.LiftoverBcfToolsWrapper; +import org.labkey.api.sequenceanalysis.run.LiftoverBcfToolsWrapper; import org.labkey.sequenceanalysis.run.util.LiftoverVcfWrapper; import org.labkey.sequenceanalysis.util.SequenceUtil; From 6040cd6df9b1df96e5559af8b586f8e675328bbb Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 1 Mar 2025 16:18:10 -0800 Subject: [PATCH 2/9] Support tool to write 10x barcodes to file --- .../singlecell/run/NimbleAlignmentStep.java | 2 ++ .../labkey/singlecell/run/NimbleAnalysis.java | 2 ++ .../labkey/singlecell/run/NimbleHelper.java | 24 +++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index 22985791e..0dcb21bf9 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -87,6 +87,8 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu } } + NimbleHelper.write10xBarcodes(localBam, getWrapper().getLogger(), rs, referenceGenome, output); + // Now run nimble itself: NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); helper.doNimbleAlign(localBam, output, rs, basename); diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java b/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java index 8610af761..8fa4390b7 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java @@ -58,6 +58,8 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); helper.doNimbleAlign(inputBam, output, rs, FileUtil.getBaseName(inputBam)); + NimbleHelper.write10xBarcodes(inputBam, getPipelineCtx().getLogger(), rs, referenceGenome, output); + return output; } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java index b48c5cb14..f26a336c7 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java @@ -5,6 +5,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.StringBuilderWriter; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.json.JSONArray; import org.json.JSONObject; @@ -27,6 +28,7 @@ import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.run.DISCVRSeqRunner; import org.labkey.api.sequenceanalysis.run.DockerWrapper; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.writer.PrintWriters; @@ -494,6 +496,28 @@ private Map doAlignment(List genomes, List barcodeArgs = new ArrayList<>(runner.getBaseArgs("Save10xBarcodes")); + barcodeArgs.add("--bam"); + barcodeArgs.add(bam.getPath()); + + File cbOutput = new File(bam.getParentFile(), SequenceAnalysisService.get().getUnzippedBaseName(bam.getName()) + "cb.txt.gz"); + barcodeArgs.add("--cbOutput"); + barcodeArgs.add(cbOutput.getPath()); + + File umiOutput = new File(bam.getParentFile(), SequenceAnalysisService.get().getUnzippedBaseName(bam.getName()) + "umi.txt.gz"); + barcodeArgs.add("--umiOutput"); + barcodeArgs.add(umiOutput.getPath()); + + runner.execute(barcodeArgs); + + output.addSequenceOutput(cbOutput, "10x CellBarcode Map: " + rs.getName(), "10x CellBarcode Map", rs.getReadsetId(), null, referenceGenome.getGenomeId(), null); + output.addSequenceOutput(umiOutput, "10x UMI Map: " + rs.getName(), "10x UMI Map", rs.getReadsetId(), null, referenceGenome.getGenomeId(), null); + } + public static File runNimbleReport(File alignResultsGz, int genomeId, PipelineStepOutput output, PipelineContext ctx) throws PipelineJobException { List reportArgs = new ArrayList<>(); From e0840868e02d1bbcbb4f02baf3e2fb78b4ab6c8f Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 3 Mar 2025 10:29:20 -0800 Subject: [PATCH 3/9] Correct argument --- singlecell/src/org/labkey/singlecell/run/NimbleHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java index f26a336c7..c33ed37c4 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java @@ -501,7 +501,7 @@ public static void write10xBarcodes(File bam, Logger log, Readset rs, ReferenceG // Write barcodes: DISCVRSeqRunner runner = new DISCVRSeqRunner(log); List barcodeArgs = new ArrayList<>(runner.getBaseArgs("Save10xBarcodes")); - barcodeArgs.add("--bam"); + barcodeArgs.add("-I"); barcodeArgs.add(bam.getPath()); File cbOutput = new File(bam.getParentFile(), SequenceAnalysisService.get().getUnzippedBaseName(bam.getName()) + "cb.txt.gz"); From e0695590dd1f0ca99bd97f5c7d51971e9de6cb4c Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 3 Mar 2025 12:42:24 -0800 Subject: [PATCH 4/9] Support SVType annotation --- .../run/variant/VariantAnnotatorStep.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/VariantAnnotatorStep.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/VariantAnnotatorStep.java index be176e657..1396e2ec6 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/VariantAnnotatorStep.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/VariantAnnotatorStep.java @@ -55,6 +55,9 @@ public Provider() ToolParameterDescriptor.create("excessHet", "ExcessHet", "If selected, the ExcessHet annotation will run.", "checkbox", new JSONObject(){{ put("checked", false); }}, null), + ToolParameterDescriptor.create("svtype", "SVType", "If selected, the SVType annotation will run.", "checkbox", new JSONObject(){{ + put("checked", false); + }}, null), new PedigreeToolParameterDescriptor() ), PageFlowUtil.set(PedigreeToolParameterDescriptor.getClientDependencyPath()), ""); } @@ -119,6 +122,12 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno options.add("ExcessHet"); } + if (getProvider().getParameterByName("svtype").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false)) + { + options.add("-A"); + options.add("SVType"); + } + if (intervals != null) { intervals.forEach(interval -> { From af0c4ba16568fa783a3e33d790add145e17cd076 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 4 Mar 2025 16:49:23 -0800 Subject: [PATCH 5/9] Bugfix to UpdateSeuratPrototype --- .../pipeline/singlecell/UpdateSeuratPrototype.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java index 690891471..f750cf2bf 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java @@ -140,7 +140,8 @@ public Output execute(SequenceOutputHandler.JobContext ctx, List Date: Fri, 7 Mar 2025 10:47:41 -0800 Subject: [PATCH 6/9] Expose option to change strandedness threshold --- .../AbstractCombineGeneCountsHandler.java | 30 ++++++++----------- .../CombineStarGeneCountsHandler.java | 18 +++++------ 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/AbstractCombineGeneCountsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/AbstractCombineGeneCountsHandler.java index 00e39ed61..7a13d0704 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/AbstractCombineGeneCountsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/AbstractCombineGeneCountsHandler.java @@ -81,11 +81,16 @@ private static List getParams(boolean allowInferStrande if (allowInferStranded) { - ret.add(ToolParameterDescriptor.create(STRANDED, "Strand 1", "Choose whether to treat these data as stranded, unstranded, or to have the script infer the strandedness", "ldk-simplecombo", new JSONObject() + ret.add(ToolParameterDescriptor.create(STRANDED, "Strandedness", "Choose whether to treat these data as stranded, unstranded, or to have the script infer the strandedness", "ldk-simplecombo", new JSONObject() {{ put("storeValues", STRAND1 + ";" + STRAND2 + ";" + UNSTRANDED + ";" + INFER); put("value", INFER); }}, INFER)); + ret.add(ToolParameterDescriptor.create("strandedThreshold", "Strandedness Threshold", "If infer is selected, Choose whether to treat these data as stranded, unstranded, or to have the script infer the strandedness", "ldk-simplecombo", new JSONObject(){{ + put("minValue", 0); + put("maxValue", 1); + put("decimalPrecision", 2); + }}, 0.9)); } return ret; @@ -305,7 +310,7 @@ public void prepareFiles(JobContext ctx, List inputFiles, St header.add(hp.getHeader(ctx, outputFileMap.get(rowId))); } - writer.writeNext(header.toArray(new String[header.size()])); + writer.writeNext(header.toArray(new String[0])); Set genesWithoutData = new TreeSet<>(); for (String geneId : results.distinctGenes) @@ -313,21 +318,12 @@ public void prepareFiles(JobContext ctx, List inputFiles, St List row = new ArrayList<>(inputFiles.size() + 3); if (translator.getGeneMap().containsKey(geneId)) { - if (translator.getGeneMap().containsKey(geneId)) - { - row.add(geneId); - row.add(translator.getGeneMap().get(geneId).get("gene_name")); - row.add(translator.getGeneMap().get(geneId).get("gene_description")); - } - else - { - row.add(geneId); - row.add(""); - row.add(""); - } + row.add(geneId); + row.add(translator.getGeneMap().get(geneId).get("gene_name")); + row.add(translator.getGeneMap().get(geneId).get("gene_description")); List toAdd = new ArrayList<>(); - Integer totalWithData = 0; + int totalWithData = 0; for (Integer rowId : outputFileMap.keySet()) { Double count = results.counts.get(rowId).get(geneId); @@ -341,9 +337,9 @@ public void prepareFiles(JobContext ctx, List inputFiles, St if (totalWithData > 0 || !doSkipGenesWithoutData) { - row.add(totalWithData.toString()); + row.add(String.valueOf(totalWithData)); row.addAll(toAdd); - writer.writeNext(row.toArray(new String[row.size()])); + writer.writeNext(row.toArray(new String[0])); } if (totalWithData == 0 && !OTHER_IDS.contains(geneId)) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineStarGeneCountsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineStarGeneCountsHandler.java index bc85902d0..7aaea197c 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineStarGeneCountsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineStarGeneCountsHandler.java @@ -31,7 +31,6 @@ public CombineStarGeneCountsHandler() @Override protected void processOutputFiles(CountResults results, List inputFiles, JSONObject params, GeneToNameTranslator translator, PipelineJob job, RecordedAction action) throws PipelineJobException { - String strandedSelection = params.optString(STRANDED, INFER); //next iterate all read count TSVs to guess strandedness @@ -69,12 +68,12 @@ protected void processOutputFiles(CountResults results, List continue; } - Long unstranded = Long.parseLong(cells[1]); - Long strand1 = Long.parseLong(cells[2]); + long unstranded = Long.parseLong(cells[1]); + long strand1 = Long.parseLong(cells[2]); totalStrand1 += strand1; - Long strand2 = Long.parseLong(cells[3]); + long strand2 = Long.parseLong(cells[3]); totalStrand2 += strand2; - Long strandMax = Math.max(strand1, strand2); + long strandMax = Math.max(strand1, strand2); results.distinctGenes.add(geneId); @@ -103,9 +102,9 @@ protected void processOutputFiles(CountResults results, List strand2Map = new HashMap<>(Math.max(results.distinctGenes.size() + 500, 5000)); } - unstrandedMap.put(geneId, unstranded.doubleValue()); - strand1Map.put(geneId, strand1.doubleValue()); - strand2Map.put(geneId, strand2.doubleValue()); + unstrandedMap.put(geneId, (double)unstranded); + strand1Map.put(geneId, (double)strand1); + strand2Map.put(geneId, (double)strand2); unstrandedCounts.put(so.getRowid(), unstrandedMap); strand1Counts.put(so.getRowid(), strand1Map); @@ -125,8 +124,9 @@ protected void processOutputFiles(CountResults results, List //finally build output double avgStrandRatio = sumStrandRatio / countStrandRatio; + double threshold = params.optDouble("strandedThreshold", 0.9); job.getLogger().info("the average stranded/unstranded ratio for all samples was: " + avgStrandRatio); - double threshold = 0.9; + String inferredStrandedness; job.getLogger().info("Attempting to infer strandedness"); if (avgStrandRatio > threshold) From 3080654edecb78cff4fb4c5cc8a5312bcc220998 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 14 Mar 2025 10:32:47 -0700 Subject: [PATCH 7/9] Auto-detect and remove '=' prefix on import --- .../resources/web/singlecell/panel/PoolImportPanel.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/singlecell/resources/web/singlecell/panel/PoolImportPanel.js b/singlecell/resources/web/singlecell/panel/PoolImportPanel.js index 8cf871dda..75bb4f4b7 100644 --- a/singlecell/resources/web/singlecell/panel/PoolImportPanel.js +++ b/singlecell/resources/web/singlecell/panel/PoolImportPanel.js @@ -182,6 +182,13 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { val = val.replace(/ PBMC/, ''); } + if (val.indexOf('=') > -1) { + const split = val.split('=') + if (split.length === 2) { + val = val[1] + } + } + return val; }, From 089d4ee6983224b9ecd95a88f6d828905047b68e Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 15 Mar 2025 07:54:14 -0700 Subject: [PATCH 8/9] Support new TCR components --- .../pipeline/singlecell/CalculateGeneComponentScores.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateGeneComponentScores.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateGeneComponentScores.java index 86f6c4586..898e2e218 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateGeneComponentScores.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateGeneComponentScores.java @@ -25,7 +25,7 @@ public Provider() super("CalculateGeneComponentScores", "Calculate Gene Module Scores", "RIRA", "This will generate UCell scores for a set of pre-defined gene modules", Collections.singletonList( SeuratToolParameter.create("savedComponent", "Saved Component(s)", "This is the name of the saved component (from RIRA) to apply", "ldk-simplecombo", new JSONObject() {{ - put("storeValues", "Tcell_NaiveToEffector;Tcell_EffectorDifferentiation"); + put("storeValues", "Tcell_EffectorDifferentiation;TCR_EarlyStimulationComponent;TCR_StimulationComponent1"); put("multiSelect", true); put("allowBlank", false); put("joinReturnValue", true); From 5ac9e24e3f87b8956c8e554fb66f908d7bd529c6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 17 Mar 2025 10:15:40 -0700 Subject: [PATCH 9/9] Bugfix to PoolImportPanel --- singlecell/resources/web/singlecell/panel/PoolImportPanel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/resources/web/singlecell/panel/PoolImportPanel.js b/singlecell/resources/web/singlecell/panel/PoolImportPanel.js index 75bb4f4b7..0d200e179 100644 --- a/singlecell/resources/web/singlecell/panel/PoolImportPanel.js +++ b/singlecell/resources/web/singlecell/panel/PoolImportPanel.js @@ -185,7 +185,7 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { if (val.indexOf('=') > -1) { const split = val.split('=') if (split.length === 2) { - val = val[1] + val = split[1] } }