From 3d7e7d8203f555649c4a659c50a3f6d0d6483300 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 25 Nov 2024 14:51:06 -0800 Subject: [PATCH 1/8] Update cluster partition name --- .../labkey/primeseq/pipeline/SequenceJobResourceAllocator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index 382fe955..be4ca8fc 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -595,7 +595,7 @@ private void possiblyAddQOS(PipelineJob job, RemoteExecutionEngine engine, List< private String getPartition(PipelineJob job) { - return needsGPUs(job) ? "gpu" : "exacloud"; + return needsGPUs(job) ? "gpu" : "batch"; } private Long getFileSize(PipelineJob job) From a0435551fe05524ad9600575dd1a244a93e2ee9c Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 26 Nov 2024 09:08:04 -0800 Subject: [PATCH 2/8] Drop sudo from docker --- primeseq/resources/external/comb-p/combpWrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primeseq/resources/external/comb-p/combpWrapper.sh b/primeseq/resources/external/comb-p/combpWrapper.sh index cfce67e4..186fed1b 100644 --- a/primeseq/resources/external/comb-p/combpWrapper.sh +++ b/primeseq/resources/external/comb-p/combpWrapper.sh @@ -18,4 +18,4 @@ DOCKER=/opt/acc/sbin/exadocker #docker build -q -t combp $1 #docker pull bbimber/combpdocker -sudo $DOCKER run --rm=true -v "${DATA_DIR}:/data" -v "${OUT_DIR}:/outDir" bbimber/combpdocker comb-p pipeline -c 5 --dist $DIST --step $STEP --seed $SEED -p /outDir/${OUT_PREFIX} /data/${INPUT_FILE} +$DOCKER run --rm=true -v "${DATA_DIR}:/data" -v "${OUT_DIR}:/outDir" bbimber/combpdocker comb-p pipeline -c 5 --dist $DIST --step $STEP --seed $SEED -p /outDir/${OUT_PREFIX} /data/${INPUT_FILE} From 5d29dc550d49f990932dc8e4b5b291990e5c5192 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 30 Nov 2024 18:15:18 -0800 Subject: [PATCH 3/8] Add exacloud scratch space --- .../org/labkey/primeseq/pipeline/ExacloudResourceSettings.java | 1 + 1 file changed, 1 insertion(+) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java index e7eaf6c7..80259277 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java @@ -55,6 +55,7 @@ public Collection getDockerVolumes(Container c) Set volumes = new HashSet<>(); volumes.add("/home/groups/prime-seq"); volumes.add("/home/exacloud/gscratch"); + volumes.add("/mnt/scratch"); PipeRoot pr = PipelineService.get().findPipelineRoot(c); if (pr != null && pr.getRootPath().exists()) From 15811a6f0016705989bb7b05f78d4149cc023147 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 3 Dec 2024 11:13:50 -0800 Subject: [PATCH 4/8] Signficiant refactor of docker in pipeline jobs to migrate everything to DockerWrapper and improve handling of input file locations --- primeseq/resources/external/comb-p/Dockerfile | 10 --- .../resources/external/comb-p/combpWrapper.sh | 21 ------ .../MethylationRateComparisonHandler.java | 38 ++++++++++- .../labkey/primeseq/pipeline/CombpRunner.java | 66 ------------------- .../pipeline/ExacloudResourceSettings.java | 37 ++++++++--- 5 files changed, 62 insertions(+), 110 deletions(-) delete mode 100644 primeseq/resources/external/comb-p/Dockerfile delete mode 100644 primeseq/resources/external/comb-p/combpWrapper.sh delete mode 100644 primeseq/src/org/labkey/primeseq/pipeline/CombpRunner.java diff --git a/primeseq/resources/external/comb-p/Dockerfile b/primeseq/resources/external/comb-p/Dockerfile deleted file mode 100644 index 47efb6ca..00000000 --- a/primeseq/resources/external/comb-p/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -from continuumio/anaconda - -RUN apt-get update --fix-missing && apt-get install -y unzip && apt-get clean - -# install comb-p -RUN mkdir -p /comb-p -RUN cd /comb-p -RUN wget https://github.com/brentp/combined-pvalues/archive/v0.32.zip -RUN unzip v0.32.zip -RUN cd /combined-pvalues-0.32 && python setup.py install diff --git a/primeseq/resources/external/comb-p/combpWrapper.sh b/primeseq/resources/external/comb-p/combpWrapper.sh deleted file mode 100644 index 186fed1b..00000000 --- a/primeseq/resources/external/comb-p/combpWrapper.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e -set -u -set -x - -DATA_DIR=$1 -INPUT_FILE=$2 -OUT_DIR=$3 -OUT_PREFIX=$4 -DIST=$5 -STEP=$6 -SEED=$7 -DOCKER=/opt/acc/sbin/exadocker - -# NOTE: switch to using dockerhub for the build -# https://github.com/bbimber/combpdocker -#docker build -q -t combp $1 -#docker pull bbimber/combpdocker - -$DOCKER run --rm=true -v "${DATA_DIR}:/data" -v "${OUT_DIR}:/outDir" bbimber/combpdocker comb-p pipeline -c 5 --dist $DIST --step $STEP --seed $SEED -p /outDir/${OUT_PREFIX} /data/${INPUT_FILE} diff --git a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java index 633a0827..e7f1e0fd 100644 --- a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java +++ b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java @@ -29,18 +29,19 @@ import org.labkey.api.security.User; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; +import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker; import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; import org.labkey.api.sequenceanalysis.run.CommandWrapper; +import org.labkey.api.sequenceanalysis.run.DockerWrapper; import org.labkey.api.util.FileType; import org.labkey.api.util.FileUtil; import org.labkey.api.util.Pair; import org.labkey.api.view.ActionURL; import org.labkey.api.writer.PrintWriters; import org.labkey.primeseq.PrimeseqModule; -import org.labkey.primeseq.pipeline.CombpRunner; import java.io.BufferedReader; import java.io.File; @@ -487,8 +488,7 @@ public void processFilesRemote(List inputFiles, JobContext c Double seed = ctx.getParams().optDouble("seed", 0.05); Integer step = ctx.getParams().optInt("step", 100); - CombpRunner combp = new CombpRunner(ctx.getLogger()); - File outBed = combp.runCompP(finalOut, ctx.getOutputDir(), dist, seed, step); + File outBed = runCompP(finalOut, ctx.getOutputDir(), dist, seed, step, ctx, ctx.getFileManager()); SequenceOutputFile so2 = new SequenceOutputFile(); so2.setName("Comb-p: " + ctx.getJob().getDescription()); so2.setDescription("Comb-p: " + jobDescription); @@ -726,4 +726,36 @@ else if (i2.compareTo(il) > 0) } } } + + private File runCompP(File inputBed, File outputDir, int dist, double seed, int stepSize, PipelineContext ctx, PipelineOutputTracker tracker) throws PipelineJobException + { + // See: https://github.com/bbimber/combpdocker + DockerWrapper wrapper = new DockerWrapper("bbimber/combpdocker", ctx.getLogger(), ctx); + File outputPrefix = new File(outputDir, FileUtil.getBaseName(inputBed) + ".combp"); + + List args = new ArrayList<>(); + args.add("comb-p"); + args.add("pipeline"); + args.add("-c"); + args.add("5"); + args.add("--dist"); + args.add(String.valueOf(dist)); + args.add("--step"); + args.add(String.valueOf(stepSize)); + args.add("--seed"); + args.add(String.valueOf(seed)); + args.add("--p"); + args.add(outputPrefix.getPath()); + args.add(inputBed.getPath()); + + wrapper.executeWithDocker(args, ctx.getWorkingDirectory(), tracker, Arrays.asList(inputBed)); + + File outputBed = new File(outputPrefix.getPath() + ".regions.bed"); + if (!outputBed.exists()) + { + throw new PipelineJobException("Unable to find expected output: " + outputBed.getPath()); + } + + return outputBed; + } } diff --git a/primeseq/src/org/labkey/primeseq/pipeline/CombpRunner.java b/primeseq/src/org/labkey/primeseq/pipeline/CombpRunner.java deleted file mode 100644 index 1deb4d0f..00000000 --- a/primeseq/src/org/labkey/primeseq/pipeline/CombpRunner.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.labkey.primeseq.pipeline; - -import org.apache.logging.log4j.Logger; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.resource.FileResource; -import org.labkey.api.resource.Resource; -import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; -import org.labkey.api.util.FileUtil; -import org.labkey.primeseq.PrimeseqModule; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -/** - * Created by bimber on 11/3/2016. - */ -public class CombpRunner extends AbstractCommandWrapper -{ - public CombpRunner(Logger log) - { - super(log); - } - - public File runCompP(File inputBed, File outputDir, int dist, double seed, int stepSize) throws PipelineJobException - { - List args = new ArrayList<>(); - File scriptFile = getScriptFile(); - args.add("bash"); - args.add(scriptFile.getPath()); - args.add(inputBed.getParentFile().getPath()); - args.add(inputBed.getName()); - File outputPrefix = new File(outputDir, FileUtil.getBaseName(inputBed) + ".combp"); - args.add(outputPrefix.getParentFile().getPath()); - args.add(outputPrefix.getName()); - args.add(String.valueOf(dist)); - args.add(String.valueOf(stepSize)); - args.add(String.valueOf(seed)); - - execute(args); - File outputBed = new File(outputPrefix.getPath() + ".regions.bed"); - if (!outputBed.exists()) - { - throw new PipelineJobException("Unable to find expected output: " + outputBed.getPath()); - } - - return outputBed; - } - - private File getScriptFile() throws PipelineJobException - { - String path = "/external/comb-p/combpWrapper.sh"; - Module module = ModuleLoader.getInstance().getModule(PrimeseqModule.NAME); - Resource script = module.getModuleResource(path); - if (script == null || !script.exists()) - throw new PipelineJobException("Unable to find file: " + path + " in module: " + PrimeseqModule.NAME); - - File f = ((FileResource) script).getFile(); - if (!f.exists()) - throw new PipelineJobException("Unable to find file: " + f.getPath()); - - return f; - } -} diff --git a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java index 80259277..38075bfb 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java @@ -1,5 +1,6 @@ package org.labkey.primeseq.pipeline; +import org.jetbrains.annotations.Nullable; import org.json.JSONObject; import org.labkey.api.data.Container; import org.labkey.api.module.ModuleLoader; @@ -9,6 +10,7 @@ import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; import org.labkey.primeseq.PrimeseqModule; +import java.io.File; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -60,11 +62,7 @@ public Collection getDockerVolumes(Container c) PipeRoot pr = PipelineService.get().findPipelineRoot(c); if (pr != null && pr.getRootPath().exists()) { - if (pr.getRootPath().getPath().startsWith("/home/groups/")) - { - String folderName = pr.getRootPath().getPath().replaceAll("^/home/groups/", "").split("/")[0]; - volumes.add("/home/groups/" + folderName); - } + volumes.add(convertHomeGroups(pr.getRootPath()).getPath()); } if (c.isWorkbook()) @@ -72,14 +70,33 @@ public Collection getDockerVolumes(Container c) PipeRoot pr2 = PipelineService.get().findPipelineRoot(c.getParent()); if (pr2 != null && pr2.getRootPath().exists()) { - if (pr2.getRootPath().getPath().startsWith("/home/groups/")) - { - String folderName = pr2.getRootPath().getPath().replaceAll("^/home/groups/", "").split("/")[0]; - volumes.add("/home/groups/" + folderName); - } + volumes.add(convertHomeGroups(pr2.getRootPath()).getPath()); } } return volumes; } + + private File convertHomeGroups(File input) + { + input = input.isDirectory() ? input : input.getParentFile(); + if (input.getPath().startsWith("/home/groups/")) + { + String folderName = input.getPath().replaceAll("^/home/groups/", "").split("/")[0]; + return new File("/home/groups/" + folderName); + } + + return input; + } + + @Override + public @Nullable File inferDockerVolume(File input) + { + if (input.getPath().startsWith("/home/exacloud/gscratch")) + { + return new File("/home/exacloud/gscratch"); + } + + return convertHomeGroups(input); + } } From 39922237f3f5721f3837affc029bc34878745d09 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 4 Dec 2024 08:02:16 -0800 Subject: [PATCH 5/8] Null check to docker volumes --- primeseq/src/org/labkey/primeseq/PrimeseqModule.java | 4 ++-- .../labkey/primeseq/pipeline/ExacloudResourceSettings.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index a50f4f91..af4095cf 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -72,8 +72,6 @@ protected void init() @Override protected void doStartupAfterSpringConfig(ModuleContext moduleContext) { - SequencePipelineService.get().registerResourceSettings(new ExacloudResourceSettings()); - SystemMaintenance.addTask(new ClusterMaintenanceTask()); ClusterService.get().registerResourceAllocator(new BlastPipelineJobResourceAllocator.Factory()); @@ -109,6 +107,8 @@ public PipelineStartup() } else { + SequencePipelineService.get().registerResourceSettings(new ExacloudResourceSettings()); + SequencePipelineService.get().registerPipelineStep(new BismarkWrapper.Provider()); SequencePipelineService.get().registerPipelineStep(new BismarkWrapper.MethylationExtractorProvider()); diff --git a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java index 38075bfb..62972422 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java @@ -83,7 +83,7 @@ private File convertHomeGroups(File input) if (input.getPath().startsWith("/home/groups/")) { String folderName = input.getPath().replaceAll("^/home/groups/", "").split("/")[0]; - return new File("/home/groups/" + folderName); + return new File("/home/groups/", folderName); } return input; @@ -92,6 +92,7 @@ private File convertHomeGroups(File input) @Override public @Nullable File inferDockerVolume(File input) { + input = input.isDirectory() ? input : input.getParentFile(); if (input.getPath().startsWith("/home/exacloud/gscratch")) { return new File("/home/exacloud/gscratch"); From 9267777912bae40c5441cde80a2da918e1e1c925 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 7 Dec 2024 11:41:50 -0800 Subject: [PATCH 6/8] Simplify merge VCF code --- .../org/labkey/mgap/pipeline/GenerateMgapTracksStep.java | 6 +++--- mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java | 4 ++-- .../org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java b/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java index 484b9c00..c1d89037 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java @@ -565,7 +565,7 @@ private Map getSampleToAlias(File input) throws PipelineJobExcep } @Override - public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException + public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException { job.getLogger().info("Merging additional track VCFs"); Map> trackToSamples = parseSampleMap(getSampleNameFile(getPipelineCtx().getSourceDirectory(true))); @@ -609,7 +609,7 @@ public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, Pi so.setCategory(TRACK_CATEGORY); so.setLibrary_id(genome.getGenomeId()); so.setDescription("mGAP track: " + trackName + ", total samples: " + trackToSamples.get(trackName).size()); - manager.addSequenceOutput(so); + ctx.getFileManager().addSequenceOutput(so); } if (getAnnotationReferenceVcf() != null) @@ -653,7 +653,7 @@ public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, Pi so.setCategory(TRACK_CATEGORY); so.setLibrary_id(genome.getGenomeId()); so.setDescription("These are novel sites in mGAP v" + releaseVersion + " for " + species); - manager.addSequenceOutput(so); + ctx.getFileManager().addSequenceOutput(so); } } diff --git a/mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java b/mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java index 1fd0e68a..05051461 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java @@ -84,7 +84,7 @@ public GroupCompareStep create(PipelineContext ctx) } @Override - public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException + public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException { job.getLogger().info("Merging variant tables"); List toConcat = orderedScatterOutputs.stream().map(f -> { @@ -138,7 +138,7 @@ public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, Pi so.setFile(combined); so.setCategory("Variant List"); so.setLibrary_id(genome.getGenomeId()); - manager.addSequenceOutput(so); + ctx.getFileManager().addSequenceOutput(so); } @Override diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java index 1caeab89..4f1929ff 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java @@ -109,7 +109,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno } @Override - public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException + public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException { job.getLogger().info("Merging missing sites VCFs"); List toConcat = orderedScatterOutputs.stream().map(f -> { @@ -142,6 +142,6 @@ public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, Pi so.setFile(combined); so.setCategory("Missing Sites VCF"); so.setLibrary_id(genome.getGenomeId()); - manager.addSequenceOutput(so); + ctx.getFileManager().addSequenceOutput(so); } } From ee64fabf1001783e47209f33d72a7d4e202a0742 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 12 Dec 2024 10:45:45 -0800 Subject: [PATCH 7/8] Update MCC packages and add color palettes --- mcc/package-lock.json | 42 +++++++++++++------- mcc/package.json | 3 +- mcc/src/client/GeneticsPlot/GeneticsPlot.tsx | 32 +++++++++++++-- mcc/src/client/GeneticsPlot/ScatterChart.tsx | 19 +++------ 4 files changed, 64 insertions(+), 32 deletions(-) diff --git a/mcc/package-lock.json b/mcc/package-lock.json index 9091b05d..7a08fcbc 100644 --- a/mcc/package-lock.json +++ b/mcc/package-lock.json @@ -15,6 +15,7 @@ "@mui/styles": "^5.14.7", "@mui/x-data-grid": "^6.0.0", "chart.js": "^4.4.4", + "google-palette": "^1.1.1", "react": "^17.0.2", "react-chartjs-2": "^5.2.0", "react-dom": "^17.0.2", @@ -5642,10 +5643,11 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -6573,9 +6575,9 @@ } }, "node_modules/express": { - "version": "4.21.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", - "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "dev": true, "license": "MIT", "dependencies": { @@ -6598,7 +6600,7 @@ "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.10", + "path-to-regexp": "0.1.12", "proxy-addr": "~2.0.7", "qs": "6.13.0", "range-parser": "~1.2.1", @@ -6613,6 +6615,10 @@ }, "engines": { "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/express/node_modules/debug": { @@ -7475,6 +7481,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/google-palette": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/google-palette/-/google-palette-1.1.1.tgz", + "integrity": "sha512-yZiM5oLl8lCZzf06IMOGdDkxqvCMd9HNFcCiOMqWgGGiGzC22vWBVhKJNvykXXbeC0NAElNH97jA/y0bq6TCrA==", + "license": "Apache-2.0" + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -10949,9 +10961,9 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", "dev": true, "funding": [ { @@ -10959,6 +10971,7 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -11560,10 +11573,11 @@ "dev": true }, "node_modules/path-to-regexp": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", - "dev": true + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "dev": true, + "license": "MIT" }, "node_modules/path-type": { "version": "4.0.0", diff --git a/mcc/package.json b/mcc/package.json index 7853ca02..bef057dc 100644 --- a/mcc/package.json +++ b/mcc/package.json @@ -23,7 +23,8 @@ "react-dom": "^17.0.2", "react-tooltip": "^5.28.0", "tsv": "^0.2.0", - "uuid": "^10.0.0" + "uuid": "^10.0.0", + "google-palette": "^1.1.1" }, "devDependencies": { "@labkey/build": "^7.7.1", diff --git a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx index 083627a1..0c007948 100644 --- a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx +++ b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx @@ -108,10 +108,34 @@ export function GeneticsPlot() { <>
- Population structure analysis using PCA is a helpful way to summarize the genetic relationships among animals in the MCC. The PCA results can be thought of as a simple type of genetic clustering - animals with more similar principal component loadings are more genetically similar. A more precise description of the relationship between two animals is provided by kinship coefficients – these are quantitative measures of relatedness that can be calculated by comparing two genomes, and interpreted using genealogical language, such as ‘parent-child’, ‘uncle-nephew’, ‘first cousins’, etc. -
-
- Whole genome sequencing was performed on each animal and genotypes were called with GATK haplotype caller. Principal components analysis was performed with GCTA (https://yanglab.westlake.edu.cn/software/gcta/#PCA) and kinship coefficients were calculated with KING (https://www.kingrelatedness.com/). Analyses were performed by Ric del Rosario (Broad Institute). + Over the past few years, the MCC team has been working on extracting, sequencing and analyzing DNA from + marmosets across the participating breeding centers. While we have deposited the raw sequence data for + 578 marmosets on NCBI's Sequence Read Archive (SRA), we are excited to report that the MCC portal now + houses a call set with single nucleotide variants and short indels for over 800 individuals. +

+ The MCC genomic database is extensive, with each individual being genotype at millions of variants + across the genome. One way to summarize a large dataset can be done using Principal Component Analysis + (PCA). PCA is a technique used across disciplines (from astronomy to genomics) that reduces the + information in a multi-dimensional dataset to (fewer) principal components (PC) that retain overall + trends and patterns in the original data. Biologically, this could mean merging together two variants + that are always inherited together into just one PC, making the data easier to analyze while maintaining + its most important patterns. See the **Visualization with PCA** tab below. +

+ Although PCA is useful for broad-scale comparisons, it is not very useful when trying to distinguish + whether two individuals are siblings or first-cousins, for instance. For that, we have better statistics + that can describe the genetic relatedness between two individuals. We estimated genetic relatedness for + all pairs of individuals for which we have whole-genome data, and made these available under the + **Kinship** tab. There you will find the inferred relationships between pairs of individuals as well as + the calculated kinship coefficient, which is a quantitative measure of genetic relatedness (see + here for more details). +

+ It is possible to explore the full MCC database of variants with a graphical interface by accessing the + **Genome Browser** tab. There you can, for example, visualize all the variants present in your gene of + interest by typing it's name in the search bar. +

+ The genetic analyses described here were performed by Karina Ray (ONPRC), Murillo Rodrigues (ONPRC), and + Ric del Rosario (Broad Institute). Please contact us at mcc@ohsu.edu with any + questions.

diff --git a/mcc/src/client/GeneticsPlot/ScatterChart.tsx b/mcc/src/client/GeneticsPlot/ScatterChart.tsx index 080c8967..a1a8490a 100644 --- a/mcc/src/client/GeneticsPlot/ScatterChart.tsx +++ b/mcc/src/client/GeneticsPlot/ScatterChart.tsx @@ -5,23 +5,14 @@ import { LineElement, Tooltip, Legend -} from 'chart.js'; +} from 'chart.js' +import palette from 'google-palette' import { Scatter } from 'react-chartjs-2'; import React, { useEffect, useRef, useState } from 'react'; ChartJS.register(LinearScale, PointElement, LineElement, Tooltip, Legend); -const CHART_COLORS = [ - 'rgb(255, 99, 132)', //red - 'rgb(255, 159, 64)', //orange - 'rgb(255, 205, 86)', //yellow - 'rgb(75, 192, 192)', //green - 'rgb(54, 162, 235)', //blue - 'rgb(153, 102, 255)', //purple - 'rgb(201, 203, 207)' //grey -] - export default function ScatterChart(props: {data: any}) { const { data } = props; @@ -39,11 +30,13 @@ export default function ScatterChart(props: {data: any}) { }); const dataByColony = [] - const uniqueColonies = [...new Set(collectedData.map(x => x.colony))] + const uniqueColonies = [...new Set(collectedData.map(x => String(x.colony)))] + const colors = palette(['Set1', 'sequential'], uniqueColonies.length); + uniqueColonies.forEach((colonyName : string, idx) => { dataByColony.push({ label: colonyName, - backgroundColor: CHART_COLORS[idx], + backgroundColor: colors[idx], data: collectedData.filter(x => x.colony == colonyName) }) }) From 0053a1d44bfb13f3fe4ac2506f72d63e61a38080 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 12 Dec 2024 12:20:30 -0800 Subject: [PATCH 8/8] Improve MCC plot colors --- mcc/src/client/GeneticsPlot/GeneticsPlot.tsx | 4 ++-- mcc/src/client/GeneticsPlot/ScatterChart.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx index 0c007948..031af83d 100644 --- a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx +++ b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx @@ -126,8 +126,8 @@ export function GeneticsPlot() { that can describe the genetic relatedness between two individuals. We estimated genetic relatedness for all pairs of individuals for which we have whole-genome data, and made these available under the **Kinship** tab. There you will find the inferred relationships between pairs of individuals as well as - the calculated kinship coefficient, which is a quantitative measure of genetic relatedness (see - here for more details). + the calculated kinship coefficient, which is a quantitative measure of genetic relatedness + (see here for more details).

It is possible to explore the full MCC database of variants with a graphical interface by accessing the **Genome Browser** tab. There you can, for example, visualize all the variants present in your gene of diff --git a/mcc/src/client/GeneticsPlot/ScatterChart.tsx b/mcc/src/client/GeneticsPlot/ScatterChart.tsx index a1a8490a..3ffd23e3 100644 --- a/mcc/src/client/GeneticsPlot/ScatterChart.tsx +++ b/mcc/src/client/GeneticsPlot/ScatterChart.tsx @@ -31,12 +31,12 @@ export default function ScatterChart(props: {data: any}) { const dataByColony = [] const uniqueColonies = [...new Set(collectedData.map(x => String(x.colony)))] - const colors = palette(['Set1', 'sequential'], uniqueColonies.length); + const colors = palette(['Set1', 'qualitative'], uniqueColonies.length); uniqueColonies.forEach((colonyName : string, idx) => { dataByColony.push({ label: colonyName, - backgroundColor: colors[idx], + backgroundColor: '#' + colors[idx], data: collectedData.filter(x => x.colony == colonyName) }) })