refactor(cli): use CCBR/Tools@v0.2.3 & update spooker usage

CHANGELOG.md

@@ -4,6 +4,14 @@
 - Fix resources for qualimap bamqc. (#79, @dnousome)
 - Now using the readthedocs theme for the docs website. (#84, @kelly-sovacool)
 - LOGAN is now archived in Zenodo with DOI `10.5281/zenodo.14907169`. (#87, @kelly-sovacool)
+- CLI updates: (#93, @kelly-sovacool)
+  - Use `nextflow run -resume` by default, or turn it off with `logan run --forceall`.
+  - Add `--output` argument for `logan init` and `logan run`.
+    - If not provided, commands are run in the current working directory.
+    - This is equivalent to the nextflow `$launchDir` constant.
+  - The nextflow preview is printed before launching the actual run.
+  - Set the `publish_dir_mode` nextflow option to `link` by default.
+  - Set the `process.cache` nextflow option to `deep` by default rather than lenient on biowulf.
 
 ## LOGAN 0.2.0
 

conf/biowulf.config

@@ -32,8 +32,7 @@ process {
     scratch = '/lscratch/$SLURM_JOBID'
     stageInMode = 'symlink'
     stageOutMode = 'rsync'
-    // for running pipeline on group sharing data directory, this can avoid inconsistent files timestamps
-    cache = 'lenient'
+    cache = 'deep'
 
     //Additional Process for BWAMEM2
     withName:bwamem2 {
@@ -65,5 +64,3 @@ process {
     }
 
 }
-
-

lib/Utils.groovy

@@ -2,14 +2,14 @@ class Utils {
     // run spooker for the workflow
     public static String spooker(workflow) {
         def pipeline_name = "${workflow.manifest.name.tokenize('/')[-1]}"
-        def command_string = "spooker ${workflow.launchDir} ${pipeline_name}"
         def out = new StringBuilder()
         def err = new StringBuilder()
         def spooker_in_path = check_command_in_path("spooker")
         if (spooker_in_path) {
             try {
                 println "Running spooker"
-                def command = command_string.execute()
+                def spooker_command = "spooker ${workflow.launchDir} ${pipeline_name} ${workflow.manifest.version}"
+                def command = spooker_command.execute()
                 command.consumeProcessOutput(out, err)
                 command.waitFor()
             } catch(IOException e) {

nextflow.config

@@ -34,7 +34,7 @@ params {
     lofreq_convert = "${projectDir}/bin/add_gt_lofreq.sh"
     strelka_convert = "${projectDir}/bin/strelka_convert.py"
     split_regions = "70" //Number of regions to split
-    
+
     vep_cache = "/fdb/VEP/102/cache"
 
     //SUB WORKFLOWS to SPLIT
@@ -45,16 +45,16 @@ params {
     snv = null
     sv = null
     structural = null
-    cnv =  null  
+    cnv =  null
     copynumber = null
-    
+
     bam=null
     align=null
     indelrealign=null
     no_tonly=null
     ffpe=null
     exome=null
-    
+
     //Set all Inputs to null
     sample_sheet=null
     fastq_file_input=null
@@ -64,15 +64,15 @@ params {
     fastq_input=null
     bam_input=null
     BAMINPUT=null
-    
+
     split_fastq = 0 //Default to no split for now but use 200000000 in future
     callers = "mutect2,deepsomatic,octopus,strelka,lofreq,muse,vardict,varscan"
     tonlycallers = "mutect2,deepsomatic,octopus,vardict,varscan"
     cnvcallers = "purple,sequenza,freec,ascat,cnvkit"
     svcallers = "manta,gridss,svaba"
     intervals = null
-    publish_dir_mode = 'symlink'
-    outdir = 'results'
+    publish_dir_mode = 'link'
+    outdir = 'output'
 
 }
 
@@ -155,6 +155,18 @@ report {
     file      = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html"
 }
 
+String pipeline_version = new File("${projectDir}/VERSION").text
+
+manifest {
+    name = "CCBR/LOGAN"
+    author = "CCR Collaborative Bioinformatics Resource"
+    homePage = "https://github.com/CCBR/LOGAN"
+    description = "whoLe genOme-sequencinG Analysis pipeliNe"
+    mainScript = "main.nf"
+    defaultBranch = "main"
+    version = "${pipeline_version}"
+}
+
 // Function to ensure that resource requirements don't go beyond
 // a maximum limit
 def check_max(obj, type) {

pyproject.toml

@@ -35,7 +35,7 @@ classifiers = [
 ]
 requires-python = ">=3.10"
 dependencies = [
-    "ccbr_tools@git+https://github.com/CCBR/Tools",
+    "ccbr_tools@git+https://github.com/CCBR/Tools@v0.2",
     "cffconvert >= 2.0.0",
     "Click >= 8.1.3",
     "pyyaml >= 6.0"

src/__main__.py

@@ -1,40 +1,44 @@
 """
 Entrypoint for LOGAN CLI
-
-Check out the wiki for a detailed look at customizing this file:
-https://github.com/beardymcjohnface/Snaketool/wiki/Customising-your-Snaketool
 """
 
-import os
 import click
-from .util import (
-    nek_base,
-    get_version,
-    copy_config,
-    OrderedCommands,
-    run_nextflow,
-    print_citation,
-)
+import os
+import pathlib
+
+import ccbr_tools.pkg_util
+import ccbr_tools.pipeline.util
+import ccbr_tools.pipeline.nextflow
 
 
-def common_options(func):
-    """Common options decorator for use with click commands."""
-    options = [
-        click.argument("nextflow_args", nargs=-1),
-    ]
-    for option in reversed(options):
-        func = option(func)
-    return func
+def repo_base(*paths):
+    basedir = pathlib.Path(__file__).absolute().parent.parent
+    return basedir.joinpath(*paths)
+
+
+def print_citation_flag(ctx, param, value):
+    if not value or ctx.resilient_parsing:
+        return
+    ccbr_tools.pkg_util.print_citation(
+        citation_file=repo_base("CITATION.cff"), output_format="bibtex"
+    )
+    ctx.exit()
 
 
 @click.group(
-    cls=OrderedCommands, context_settings=dict(help_option_names=["-h", "--help"])
+    cls=ccbr_tools.pkg_util.CustomClickGroup,
+    context_settings=dict(help_option_names=["-h", "--help"]),
+)
+@click.version_option(
+    ccbr_tools.pkg_util.get_version(repo_base=repo_base),
+    "-v",
+    "--version",
+    is_flag=True,
 )
-@click.version_option(get_version(), "-v", "--version", is_flag=True)
 @click.option(
     "--citation",
     is_flag=True,
-    callback=print_citation,
+    callback=print_citation_flag,
     expose_value=False,
     is_eager=True,
     help="Print the citation in bibtex format and exit.",
@@ -63,6 +67,8 @@ def cli():
 """
 
 
+# DEVELOPER NOTE: cannot use single-hyphen options e.g. -m, -o or else it may clash with nextflow's cli options
+# e.g. -profile clashed with -o (--output) and caused the command to be parsed as "-pr -o file"
 @click.command(
     epilog=help_msg_extra,
     context_settings=dict(
@@ -74,9 +80,16 @@ def cli():
     "main_path",
     help="Path to the logan main.nf file or the GitHub repo (CCBR/LOGAN). Defaults to the version installed in the $PATH.",
     type=str,
-    default=nek_base(os.path.join("main.nf")),
+    default=repo_base("main.nf"),
     show_default=True,
 )
+@click.option(
+    "--output",
+    help="Output directory path for logan init & run. Equivalient to nextflow launchDir. Defaults to your current working directory.",
+    type=click.Path(file_okay=False, dir_okay=True, writable=True),
+    default=pathlib.Path.cwd(),
+    show_default=False,
+)
 @click.option(
     "--mode",
     "_mode",
@@ -85,8 +98,17 @@ def cli():
     default="local",
     show_default=True,
 )
-@common_options
-def run(main_path, _mode, **kwargs):
+@click.option(
+    "--forceall",
+    "-F",
+    "force_all",
+    help="Force all processes to run (i.e. do not use nextflow -resume)",
+    is_flag=True,
+    default=False,
+    show_default=True,
+)
+@click.argument("nextflow_args", nargs=-1)
+def run(main_path, output, _mode, force_all, **kwargs):
     """Run the workflow"""
     if (  # this is the only acceptable github repo option for logan
         main_path != "CCBR/LOGAN"
@@ -96,21 +118,41 @@ def run(main_path, _mode, **kwargs):
             raise FileNotFoundError(
                 f"Path to the logan main.nf file not found: {main_path}"
             )
-
-    run_nextflow(
-        nextfile_path=main_path,
-        mode=_mode,
-        **kwargs,
-    )
+    output_dir = output if isinstance(output, pathlib.Path) else pathlib.Path(output)
+    ccbr_tools.pkg_util.msg_box("Output Directory", errmsg=str(output_dir))
+    if not output_dir.is_dir() or not (output_dir / "nextflow.config").exists():
+        raise FileNotFoundError(
+            f"output directory not initialized: {output_dir}. Hint: you must initialize the output directory with `logan init --output {output_dir}`"
+        )
+    current_wd = os.getcwd()
+    try:
+        os.chdir(output_dir)
+        ccbr_tools.pipeline.nextflow.run(
+            nextfile_path=main_path,
+            mode=_mode,
+            force_all=force_all,
+            pipeline_name="LOGAN",
+            **kwargs,
+        )
+    finally:
+        os.chdir(current_wd)
 
 
 @click.command()
-def init(**kwargs):
-    """Initialize the working directory by copying the system default config files"""
+@click.option(
+    "--output",
+    help="Output directory path for logan init & run. Equivalient to nextflow launchDir. Defaults to your current working directory.",
+    type=click.Path(file_okay=False, dir_okay=True, writable=True),
+    default=pathlib.Path.cwd(),
+    show_default=False,
+)
+def init(output):
+    """Initialize the launch directory by copying the system default config files"""
+    output_dir = output if isinstance(output, pathlib.Path) else pathlib.Path(output)
+    ccbr_tools.pkg_util.msg_box(f"Initializing LOGAN in {output_dir}")
+    (output_dir / "log/").mkdir(parents=True, exist_ok=True)
     paths = ("nextflow.config", "conf/", "assets/")
-    copy_config(paths)
-    if not os.path.exists("log/"):
-        os.mkdir("log/")
+    ccbr_tools.pipeline.util.copy_config(paths, repo_base=repo_base, outdir=output_dir)
 
 
 cli.add_command(run)
@@ -121,5 +163,7 @@ def main():
     cli()
 
 
+cli(prog_name="logan")
+
 if __name__ == "__main__":
     main()