submission fixes for upgrade to snakemake 8+

Author: ctastad

workflow/Snakefile

@@ -31,9 +31,9 @@ rule to_spatialdata:
         paths.sdata_zgroup if paths.data_path else [],
     conda:
         "sopa2"
-    threads: 8
+    threads: 2
     resources:
-        mem_mb=128_000,
+        lsf_extra="-gpu num=1"
     params:
         reader = args['read'].as_cli(),
         data_path = paths.data_path,
@@ -50,7 +50,7 @@ rule tissue_segmentation:
         touch(paths.smk_tissue_segmentation),
     conda:
         "sopa2"
-    threads: 8
+    threads: 2
     params:
         tissue_segmentation = args["segmentation"]["tissue"].as_cli(),
         sdata_path = paths.sdata_path,
@@ -71,7 +71,7 @@ checkpoint patchify_image:
         sdata_path = paths.sdata_path,
     conda:
         "sopa2"
-    threads: 8
+    threads: 2
     shell:
         """
         sopa patchify image {params.sdata_path} {params.patchify_image}
@@ -90,9 +90,7 @@ checkpoint patchify_transcripts:
         sdata_path = paths.sdata_path,
     conda:
         "sopa2"
-    threads: 8
-    resources:
-        mem_mb=32_000,
+    threads: 2
     shell:
         """
         sopa patchify transcripts {params.sdata_path} {params.patchify_transcripts}
@@ -106,11 +104,9 @@ rule patch_segmentation_cellpose:
         paths.smk_cellpose_temp_dir / "{index}.parquet",
     conda:
         "sopa2"
-    threads: 1
+    threads: 2
     resources:
-        mem_mb=32_000,
-        queue="gpu -R 'h100nvl' -gpu 'num=1'",
-        time=45,
+        lsf_extra="-gpu num=1"
     params:
         cellpose = args["segmentation"]["cellpose"].as_cli(),
         sdata_path = paths.sdata_path,
@@ -129,8 +125,6 @@ rule patch_segmentation_baysor:
     params:
         baysor_config = args["segmentation"]["baysor"].as_cli(keys=["config"]),
         sdata_path = paths.sdata_path,
-    resources:
-        cpus_per_task=8,
     shell:
         """
         export JULIA_NUM_THREADS={resources.cpus_per_task} # parallelize within each patch for Baysor >= v0.7
@@ -140,7 +134,7 @@ rule patch_segmentation_baysor:
             module purge
         fi
 
-        sopa segmentation baysor {params.sdata_path} --patch-index {wildcards.index} {params.baysor_config}
+        sopa segmentation baysor {params.sdata_path} --patch-index {wildcards.index} {params.baysor_config} --force
         """
 
 rule patch_segmentation_comseg:
@@ -151,9 +145,6 @@ rule patch_segmentation_comseg:
         paths.smk_transcripts_temp_dir / "{index}" / "segmentation_counts.h5ad",
     conda:
         "sopa2"
-    threads: 8
-    resources:
-        mem_mb=128_000,
     params:
         comseg_config = args["segmentation"]["comseg"].as_cli(keys=["config"]),
         sdata_path = paths.sdata_path,
@@ -176,7 +167,7 @@ rule resolve_cellpose:
         touch(paths.smk_cellpose_boundaries),
     conda:
         "sopa2"
-    threads: 8
+    threads: 2
     params:
         sdata_path = paths.sdata_path,
     shell:
@@ -192,7 +183,7 @@ rule resolve_baysor:
         touch(paths.smk_table),
     conda:
         "sopa2"
-    threads: 8
+    threads: 2
     params:
         resolve = args.resolve_transcripts(),
         sdata_path = paths.sdata_path,
@@ -212,7 +203,7 @@ rule resolve_comseg:
         touch(paths.smk_table),
     conda:
         "sopa2"
-    threads: 8
+    threads: 2
     params:
         resolve = args.resolve_transcripts(),
         sdata_path = paths.sdata_path,
@@ -241,9 +232,7 @@ rule aggregate:
         touch(paths.smk_aggregation),
     conda:
         "sopa2"
-    threads: 8
-    resources:
-        mem_mb=64_000,
+    threads: 2
     params:
         aggregate = args["aggregate"].as_cli(),
         sdata_path = paths.sdata_path,
@@ -261,9 +250,7 @@ rule annotate:
         "sopa2"
     threads: 2
     resources:
-        mem_mb=64_000,
-        queue="gpu -R 'h100nvl' -gpu 'num=1'",
-        time=180,
+        lsf_extra="-gpu num=1"
     params:
         method_name = args['annotation']['method'],
         annotation = args['annotation']['args'].as_cli() + " --save-maps",
@@ -282,9 +269,7 @@ rule novae:
         "novae"
     threads: 16
     resources:
-        mem_mb=64_000,
-        queue="gpu -R 'h100nvl' -gpu 'num=1'",
-        time=45,
+        lsf_extra="-gpu num=1"
     params:
         sdata_path = paths.sdata_path,
         radius = config["novae"]["radius"],
@@ -304,9 +289,7 @@ rule image_write:
         paths.explorer_image,
     conda:
         "sopa2"
-    threads: 8
-    resources:
-        mem_mb=64_000,
+    threads: 2
     params:
         explorer = args["explorer"].as_cli(keys=['lazy', 'ram_threshold_gb', 'pixel_size', 'pixelsize']),
         sdata_path = paths.sdata_path,
@@ -329,9 +312,7 @@ rule report:
         report = paths.report,
     conda:
         "sopa2"
-    threads: 8
-    resources:
-        mem_mb=32_000,
+    threads: 2
     shell:
         """
         sopa report {params.sdata_path} {params.report}
@@ -347,9 +328,7 @@ rule explorer:
         paths.explorer_experiment,
     conda:
         "sopa2"
-    threads: 8
-    resources:
-        mem_mb=256_000,
+    threads: 2
     params:
         explorer = args["explorer"].as_cli(),
         sdata_path = paths.sdata_path,

workflow/lsf/config.yaml

@@ -1,17 +1,46 @@
-# Warning: this profile is still experimental.
-
 executor: lsf
 default-resources:
-    mem_mb: 8_000 # default rules memory in MB (you can keep this value)
+    mem_mb: 8_000
     gpu: 0
+    lsf_project: acc_untreatedIBD
+    lsf_queue: premium
+    runtime: 60
+
+set-resources:
+    to_spatialdata:
+        mem_mb: 128_000
+    image_write:
+        mem_mb: 64_000
+    patchify_transcripts:
+        mem_mb: 32_000
+    patch_segmentation_cellpose:
+        lsf_queue: gpu
+        mem_mb: 32_000
+        runtime: 45
+    annotate:
+        lsf_queue: gpu
+        mem_mb: 64_000
+        runtime: 180
+    novae:
+        lsf_queue: gpu
+        mem_mb: 64_000
+        runtime: 45
+    patch_segmentation_baysor:
+        mem_mb: 32_000
+        cpus_per_task: 2
+    resolve_baysor:
+        mem_mb: 128_000
+    aggregate:
+        mem_mb: 64_000
+    report:
+        mem_mb: 32_000
+    explorer:
+        mem_mb: 256_000
 
-# other options (you can keep the default values)
-restart-times: 0
-max-jobs-per-second: 100
-max-status-checks-per-second: 1
-local-cores: 1
-latency-wait: 60
-jobs: 50
+restart-times: 3
+max-jobs-per-second: 10
+jobs: 20000
+latency-wait: 2
 keep-going: True
 rerun-incomplete: True
 printshellcmds: True