From d4f2ec342bd49eac6243b9f8c89b8e913542df63 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Wed, 6 May 2026 08:55:02 +0100 Subject: [PATCH 1/7] feat(template): add axis-decomposed process_cpus_* / process_mem_* labels Introduces a parallel set of resource labels that decompose along the cpu and memory axes, so a process can express its shape independently: process_cpus_{single,low,medium,high} - cpus only process_mem_{low,medium,high} - memory only Stack one of each (and optionally a time-only label like process_long) on a process. The combined labels (process_single, process_low, process_medium, process_high, process_high_memory) stay in place for backwards compatibility while the ecosystem migrates; deprecation will follow in a separate PR once enough downstream pipelines / configs have adopted the new scheme. Why: the existing combined labels couple cpus and memory at fixed ratios (e.g. process_high = 12 cpus + 72 GB), which is wrong for tools that are cpu-bound but memory-light (Rust streaming binaries, e.g. trim_galore 2.x at ~100 MB peak_rss) or vice versa. Splitting the axes lets module authors pick each independently and stops over-allocating on one axis to get headroom on the other. --- CHANGELOG.md | 4 +++ nf_core/components/create.py | 9 ++++++ nf_core/modules/lint/main_nf.py | 9 ++++++ nf_core/pipeline-template/conf/base.config | 34 ++++++++++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 958446161f..7bb0904ae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,16 @@ ### Linting +- accept axis-decomposed `process_cpus_*` and `process_mem_*` labels alongside the legacy combined labels ([#XXXX](https://github.com/nf-core/tools/pull/XXXX)) + ### Modules ### Subworkflows ### Template +- add axis-decomposed `process_cpus_{single,low,medium,high}` and `process_mem_{low,medium,high}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#XXXX](https://github.com/nf-core/tools/pull/XXXX)) + #### Version updates ## [v4.0.2 - Bold Boa Patch 2](https://github.com/nf-core/tools/releases/tag/4.0.2) - [2026-04-30] diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 334278efbc..ab984050a6 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -244,6 +244,15 @@ def _get_bioconda_tool(self): def _get_module_structure_components(self): process_label_defaults = [ + # Axis-decomposed labels (preferred) + "process_cpus_single", + "process_cpus_low", + "process_cpus_medium", + "process_cpus_high", + "process_mem_low", + "process_mem_medium", + "process_mem_high", + # Legacy combined labels (kept for backwards compatibility, to be deprecated) "process_single", "process_low", "process_medium", diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index eb365a4d73..d592fd3b54 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -616,6 +616,15 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): def check_process_labels(self, lines): correct_process_labels = [ + # Axis-decomposed labels (preferred) + "process_cpus_single", + "process_cpus_low", + "process_cpus_medium", + "process_cpus_high", + "process_mem_low", + "process_mem_medium", + "process_mem_high", + # Legacy combined labels (kept for backwards compatibility, to be deprecated) "process_single", "process_low", "process_medium", diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 0edc34436d..9c4e9b9f7c 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -26,6 +26,40 @@ process { // adding in your local modules too. // TODO nf-core: Customise requirements for specific processes. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors + + // + // Axis-decomposed labels (preferred). Stack one cpus_* and one mem_* label + // (and optionally a time-only label) on a process to express its resource + // shape independently along each axis. e.g. a cpu-bound, memory-light tool + // can use `process_cpus_high` + `process_mem_low`. + // + withLabel:process_cpus_single { + cpus = { 1 } + } + withLabel:process_cpus_low { + cpus = { 2 * task.attempt } + } + withLabel:process_cpus_medium { + cpus = { 6 * task.attempt } + } + withLabel:process_cpus_high { + cpus = { 12 * task.attempt } + } + withLabel:process_mem_low { + memory = { 1.GB * task.attempt } + } + withLabel:process_mem_medium { + memory = { 12.GB * task.attempt } + } + withLabel:process_mem_high { + memory = { 72.GB * task.attempt } + } + + // + // Legacy combined labels. Kept for backwards compatibility while the + // ecosystem migrates to the axis-decomposed labels above; these will be + // deprecated in a future release. + // withLabel:process_single { cpus = { 1 } memory = { 6.GB * task.attempt } From 3614bcb3b61df8bb825d2c3cdd2fb1d41b666e11 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Wed, 6 May 2026 08:55:53 +0100 Subject: [PATCH 2/7] docs(changelog): fill in PR number --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bb0904ae2..fe0eba0710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ### Linting -- accept axis-decomposed `process_cpus_*` and `process_mem_*` labels alongside the legacy combined labels ([#XXXX](https://github.com/nf-core/tools/pull/XXXX)) +- accept axis-decomposed `process_cpus_*` and `process_mem_*` labels alongside the legacy combined labels ([#4265](https://github.com/nf-core/tools/pull/4265)) ### Modules @@ -14,7 +14,7 @@ ### Template -- add axis-decomposed `process_cpus_{single,low,medium,high}` and `process_mem_{low,medium,high}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#XXXX](https://github.com/nf-core/tools/pull/XXXX)) +- add axis-decomposed `process_cpus_{single,low,medium,high}` and `process_mem_{low,medium,high}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#4265](https://github.com/nf-core/tools/pull/4265)) #### Version updates From 978e720aabf90131bc495b111b43741921f20e8f Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Wed, 6 May 2026 09:01:12 +0100 Subject: [PATCH 3/7] feat(template): add process_time_{short,medium,long} time axis Completes the axis-decomposed scheme: alongside cpus_* and mem_*, a process can now also pin its time budget independently. Values: process_time_short = 1.h * task.attempt process_time_medium = 8.h * task.attempt process_time_long = 20.h * task.attempt short = 1.h is a meaningful step down from the 4.h template default for fast tools (streaming utilities, QC, samtools view), and lets schedulers route them to short-queue priority pools. medium / long match the existing process_medium and process_long values exactly, so module authors migrating from those don't change their effective time budget. The existing process_long label stays in the legacy block; it has the same value as process_time_long but removing it would break pipelines, so it deprecates on the same slow-deprecation timeline as the other combined labels. --- CHANGELOG.md | 4 ++-- nf_core/components/create.py | 3 +++ nf_core/modules/lint/main_nf.py | 3 +++ nf_core/pipeline-template/conf/base.config | 17 +++++++++++++---- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe0eba0710..236011b710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ### Linting -- accept axis-decomposed `process_cpus_*` and `process_mem_*` labels alongside the legacy combined labels ([#4265](https://github.com/nf-core/tools/pull/4265)) +- accept axis-decomposed `process_cpus_*`, `process_mem_*` and `process_time_*` labels alongside the legacy combined labels ([#4265](https://github.com/nf-core/tools/pull/4265)) ### Modules @@ -14,7 +14,7 @@ ### Template -- add axis-decomposed `process_cpus_{single,low,medium,high}` and `process_mem_{low,medium,high}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#4265](https://github.com/nf-core/tools/pull/4265)) +- add axis-decomposed `process_cpus_{single,low,medium,high}`, `process_mem_{low,medium,high}` and `process_time_{short,medium,long}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#4265](https://github.com/nf-core/tools/pull/4265)) #### Version updates diff --git a/nf_core/components/create.py b/nf_core/components/create.py index ab984050a6..c14e9a1e27 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -252,6 +252,9 @@ def _get_module_structure_components(self): "process_mem_low", "process_mem_medium", "process_mem_high", + "process_time_short", + "process_time_medium", + "process_time_long", # Legacy combined labels (kept for backwards compatibility, to be deprecated) "process_single", "process_low", diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index d592fd3b54..17c11d5ee4 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -624,6 +624,9 @@ def check_process_labels(self, lines): "process_mem_low", "process_mem_medium", "process_mem_high", + "process_time_short", + "process_time_medium", + "process_time_long", # Legacy combined labels (kept for backwards compatibility, to be deprecated) "process_single", "process_low", diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 9c4e9b9f7c..39bc658073 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -28,10 +28,10 @@ process { // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors // - // Axis-decomposed labels (preferred). Stack one cpus_* and one mem_* label - // (and optionally a time-only label) on a process to express its resource - // shape independently along each axis. e.g. a cpu-bound, memory-light tool - // can use `process_cpus_high` + `process_mem_low`. + // Axis-decomposed labels (preferred). Stack one cpus_*, one mem_*, and + // one time_* label on a process to express its resource shape + // independently along each axis. e.g. a cpu-bound, memory-light, fast + // tool can use `process_cpus_high` + `process_mem_low` + `process_time_short`. // withLabel:process_cpus_single { cpus = { 1 } @@ -54,6 +54,15 @@ process { withLabel:process_mem_high { memory = { 72.GB * task.attempt } } + withLabel:process_time_short { + time = { 1.h * task.attempt } + } + withLabel:process_time_medium { + time = { 8.h * task.attempt } + } + withLabel:process_time_long { + time = { 20.h * task.attempt } + } // // Legacy combined labels. Kept for backwards compatibility while the From 29e03af3d787e2437b7c7f54a5cdc436b00b820d Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 13 May 2026 15:57:44 +0200 Subject: [PATCH 4/7] add deprecation warning and handle multiple labels correctly --- nf_core/modules/lint/main_nf.py | 38 +++++++++++++++++++++++++----- tests/modules/lint/test_main_nf.py | 30 +++++++++++++---------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index a33eb3931b..1426965f70 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -627,7 +627,8 @@ def check_process_labels(self, lines): "process_time_short", "process_time_medium", "process_time_long", - # Legacy combined labels (kept for backwards compatibility, to be deprecated) + ] + legacy_process_labels = [ "process_single", "process_low", "process_medium", @@ -639,6 +640,7 @@ def check_process_labels(self, lines): all_labels = [line.strip() for line in lines if line.lstrip().startswith("label ")] bad_labels = [] good_labels = [] + legacy_labels = [] if len(all_labels) > 0: for label in all_labels: try: @@ -653,23 +655,47 @@ def check_process_labels(self, lines): ) ) continue - if label not in correct_process_labels: + if label not in correct_process_labels and label not in legacy_process_labels: bad_labels.append(label) + if label in legacy_process_labels: + legacy_labels.append(label) else: good_labels.append(label) - if len(good_labels) > 1: + axes = [label.split("_")[1] for label in good_labels if len(label.split("_")) > 1] + if len(axes) != len(set(axes)): + conflicting = [ + label + for label in good_labels + if axes.count(label.split("_")[1] if len(label.split("_")) > 1 else "") > 1 + ] self.warned.append( ( "main_nf", "process_standard_label", - f"Conflicting process labels found: `{'`,`'.join(good_labels)}`", + f"Conflicting process labels found: `{'`,`'.join(conflicting)}`", + self.main_nf, + ) + ) + elif good_labels: + self.passed.append( + ( + "main_nf", + "process_standard_label", + f"Correct process labels: `{'`,`'.join(good_labels)}`", self.main_nf, ) ) - elif len(good_labels) == 1: - self.passed.append(("main_nf", "process_standard_label", "Correct process label", self.main_nf)) else: self.warned.append(("main_nf", "process_standard_label", "Standard process label not found", self.main_nf)) + if legacy_labels: + self.warned.append( + ( + "main_nf", + "process_standard_label", + f"Deprecated process labels found: `{'`,`'.join(legacy_labels)}`. Use the new standard labels instead: https://nf-co.re/docs/developing/migration-guides/resource-labels", + self.main_nf, + ) + ) if len(bad_labels) > 0: self.warned.append( ( diff --git a/tests/modules/lint/test_main_nf.py b/tests/modules/lint/test_main_nf.py index f6c3848f71..9b64b33e0e 100644 --- a/tests/modules/lint/test_main_nf.py +++ b/tests/modules/lint/test_main_nf.py @@ -17,19 +17,25 @@ @pytest.mark.parametrize( "content,passed,warned,failed", [ - # Valid process label - ("label 'process_high'\ncpus 12", 1, 0, 0), - # Non-alphanumeric characters in label + # Valid new-style axis-decomposed label + ("label 'process_cpus_high'\ncpus 12", 1, 0, 0), + # Multiple axis-decomposed labels on different axes — allowed + ("label 'process_cpus_high'\nlabel 'process_mem_low'\ncpus 12", 1, 0, 0), + # Same axis twice — conflict + ("label 'process_cpus_high'\nlabel 'process_cpus_low'\ncpus 12", 0, 1, 0), + # Legacy label warns (no new-style label present, so also warns "not found") + ("label 'process_high'\ncpus 12", 0, 2, 0), + # Two legacy labels warn legacy + not-found (distinct, so no dup warn) + ("label 'process_high'\nlabel 'process_low'\ncpus 12", 0, 2, 0), + # Duplicate legacy labels: not-found + legacy + duplicate + ("label 'process_high'\nlabel 'process_high'\ncpus 12", 0, 3, 0), + # Non-alphanumeric characters in label: non-alphanumeric warn + not-found warn ("label 'a:label:with:colons'\ncpus 12", 0, 2, 0), - # Conflicting labels - ("label 'process_high'\nlabel 'process_low'\ncpus 12", 0, 1, 0), - # Duplicate labels - ("label 'process_high'\nlabel 'process_high'\ncpus 12", 0, 2, 0), - # Valid and non-standard labels - ("label 'process_high'\nlabel 'process_extra_label'\ncpus 12", 1, 1, 0), - # Non-standard label only - ("label 'process_extra_label'\ncpus 12", 0, 2, 0), - # Non-standard duplicates without quotes + # Non-standard (bad) label only: passes axis check but warns non-standard + ("label 'process_extra_label'\ncpus 12", 1, 1, 0), + # Legacy + non-standard: passes axis check, warns legacy and non-standard + ("label 'process_high'\nlabel 'process_extra_label'\ncpus 12", 1, 2, 0), + # Non-standard duplicates without quotes: conflict + non-standard + duplicate ("label process_extra_label\nlabel process_extra_label\ncpus 12", 0, 3, 0), # No label found ("cpus 12", 0, 1, 0), From bb54343351a0371a4ea656ebd57a505b86cda4e7 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 13 May 2026 15:58:07 +0200 Subject: [PATCH 5/7] fix pyright warnings --- nf_core/modules/lint/main_nf.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 1426965f70..4c32a2a9de 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -12,7 +12,7 @@ import yaml from rich.progress import Progress -import nf_core +import nf_core.utils from nf_core.components.components_differ import ComponentsDiffer from nf_core.components.nfcore_component import NFCoreComponent @@ -57,7 +57,26 @@ def main_nf( equal the number of ``emit:`` outputs whose name starts with ``versions``. A warning is issued if a legacy YAML-based ``versions`` emit is used instead of a topic output. - + * ``process_standard_label``: Process labels should follow the standard format. + A warning is issued if a legacy label is used. Allowed standard labels are: + "process_cpus_single", + "process_cpus_low", + "process_cpus_medium", + "process_cpus_high", + "process_mem_low", + "process_mem_medium", + "process_mem_high", + "process_time_short", + "process_time_medium", + "process_time_long" + Legacy labels are: + "process_single", + "process_low", + "process_medium", + "process_high", + "process_long", + "process_low_memory", + "process_high_memory" """ inputs: list[str] = [] @@ -643,9 +662,8 @@ def check_process_labels(self, lines): legacy_labels = [] if len(all_labels) > 0: for label in all_labels: - try: - label = re.match(r"^label\s+'?\"?([a-zA-Z0-9_-]+)'?\"?$", label).group(1) - except AttributeError: + match = re.match(r"^label\s+'?\"?([a-zA-Z0-9_-]+)'?\"?$", label) + if match is None: self.warned.append( ( "main_nf", @@ -655,6 +673,7 @@ def check_process_labels(self, lines): ) ) continue + label = match.group(1) if label not in correct_process_labels and label not in legacy_process_labels: bad_labels.append(label) if label in legacy_process_labels: From b1f31de1d69984e3fbbf20a8261041754aecf34e Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 13 May 2026 16:06:02 +0200 Subject: [PATCH 6/7] fix warning message order --- nf_core/modules/lint/main_nf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 4c32a2a9de..0ef6864b14 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -704,17 +704,17 @@ def check_process_labels(self, lines): self.main_nf, ) ) - else: - self.warned.append(("main_nf", "process_standard_label", "Standard process label not found", self.main_nf)) - if legacy_labels: + elif legacy_labels: self.warned.append( ( "main_nf", "process_standard_label", - f"Deprecated process labels found: `{'`,`'.join(legacy_labels)}`. Use the new standard labels instead: https://nf-co.re/docs/developing/migration-guides/resource-labels", + f"Deprecated process label found: `{legacy_labels[0]}`. Use the new standard labels instead: https://nf-co.re/docs/developing/migration-guides/resource-labels", self.main_nf, ) ) + else: + self.warned.append(("main_nf", "process_standard_label", "Standard process label not found", self.main_nf)) if len(bad_labels) > 0: self.warned.append( ( @@ -733,6 +733,7 @@ def check_process_labels(self, lines): self.main_nf, ) ) + else: self.warned.append(("main_nf", "process_standard_label", "Process label not specified", self.main_nf)) From 44476cae2dc7a42b7c9768a8aae2813d0043b60f Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 13 May 2026 17:12:21 +0200 Subject: [PATCH 7/7] fix test logic --- nf_core/modules/lint/main_nf.py | 18 +++++++++--------- tests/modules/lint/test_main_nf.py | 6 +++++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0ef6864b14..7cd31ce21e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -680,6 +680,15 @@ def check_process_labels(self, lines): legacy_labels.append(label) else: good_labels.append(label) + if legacy_labels: + self.warned.append( + ( + "main_nf", + "process_standard_label", + f"Deprecated process label found: `{legacy_labels[0]}`. Use the new standard labels instead: https://nf-co.re/docs/developing/migration-guides/resource-labels", + self.main_nf, + ) + ) axes = [label.split("_")[1] for label in good_labels if len(label.split("_")) > 1] if len(axes) != len(set(axes)): conflicting = [ @@ -704,15 +713,6 @@ def check_process_labels(self, lines): self.main_nf, ) ) - elif legacy_labels: - self.warned.append( - ( - "main_nf", - "process_standard_label", - f"Deprecated process label found: `{legacy_labels[0]}`. Use the new standard labels instead: https://nf-co.re/docs/developing/migration-guides/resource-labels", - self.main_nf, - ) - ) else: self.warned.append(("main_nf", "process_standard_label", "Standard process label not found", self.main_nf)) if len(bad_labels) > 0: diff --git a/tests/modules/lint/test_main_nf.py b/tests/modules/lint/test_main_nf.py index 9b64b33e0e..ec5c3229a8 100644 --- a/tests/modules/lint/test_main_nf.py +++ b/tests/modules/lint/test_main_nf.py @@ -164,7 +164,11 @@ def test_topics_and_emits_version_check(self): module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) module_lint.lint(print_results=False, module="bamstats/generalstats") assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" - assert len(module_lint.warned) == 0, f"Expected 0 warnings, got {[x.__dict__ for x in module_lint.warned]}" + # TODO: once modules are migrated to axis-decomposed labels, remove the filter and assert len(module_lint.warned) == 0 + non_label_warned = [w for w in module_lint.warned if w.lint_test != "process_standard_label"] + assert len(non_label_warned) == 0, ( + f"Expected 0 non-label warnings, got {[x.__dict__ for x in non_label_warned]}" + ) assert len(module_lint.passed) > 0