diff --git a/CHANGELOG.md b/CHANGELOG.md index 56398753fc..43ce062cc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Linting +- accept axis-decomposed `process_cpus_*`, `process_mem_*` and `process_time_*` labels alongside the legacy combined labels ([#4265](https://github.com/nf-core/tools/pull/4265)) - accept `process_low_memory` as a standard module label ([#4264](https://github.com/nf-core/tools/pull/4264)) ### Modules @@ -17,6 +18,7 @@ ### Template +- add axis-decomposed `process_cpus_{single,low,medium,high}`, `process_mem_{low,medium,high}` and `process_time_{short,medium,long}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#4265](https://github.com/nf-core/tools/pull/4265)) - add `process_low_memory` resource label to `base.config` ([#4264](https://github.com/nf-core/tools/pull/4264)) #### Version updates diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 270cf1355a..f860cdc006 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -244,6 +244,18 @@ def _get_bioconda_tool(self): def _get_module_structure_components(self): process_label_defaults = [ + # Axis-decomposed labels (preferred) + "process_cpus_single", + "process_cpus_low", + "process_cpus_medium", + "process_cpus_high", + "process_mem_low", + "process_mem_medium", + "process_mem_high", + "process_time_short", + "process_time_medium", + "process_time_long", + # Legacy combined labels (kept for backwards compatibility, to be deprecated) "process_single", "process_low", "process_medium", diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index b589bc3f0c..7cd31ce21e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -12,7 +12,7 @@ import yaml from rich.progress import Progress -import nf_core +import nf_core.utils from nf_core.components.components_differ import ComponentsDiffer from nf_core.components.nfcore_component import NFCoreComponent @@ -57,7 +57,26 @@ def main_nf( equal the number of ``emit:`` outputs whose name starts with ``versions``. A warning is issued if a legacy YAML-based ``versions`` emit is used instead of a topic output. - + * ``process_standard_label``: Process labels should follow the standard format. + A warning is issued if a legacy label is used. Allowed standard labels are: + "process_cpus_single", + "process_cpus_low", + "process_cpus_medium", + "process_cpus_high", + "process_mem_low", + "process_mem_medium", + "process_mem_high", + "process_time_short", + "process_time_medium", + "process_time_long" + Legacy labels are: + "process_single", + "process_low", + "process_medium", + "process_high", + "process_long", + "process_low_memory", + "process_high_memory" """ inputs: list[str] = [] @@ -616,6 +635,19 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): def check_process_labels(self, lines): correct_process_labels = [ + # Axis-decomposed labels (preferred) + "process_cpus_single", + "process_cpus_low", + "process_cpus_medium", + "process_cpus_high", + "process_mem_low", + "process_mem_medium", + "process_mem_high", + "process_time_short", + "process_time_medium", + "process_time_long", + ] + legacy_process_labels = [ "process_single", "process_low", "process_medium", @@ -627,11 +659,11 @@ def check_process_labels(self, lines): all_labels = [line.strip() for line in lines if line.lstrip().startswith("label ")] bad_labels = [] good_labels = [] + legacy_labels = [] if len(all_labels) > 0: for label in all_labels: - try: - label = re.match(r"^label\s+'?\"?([a-zA-Z0-9_-]+)'?\"?$", label).group(1) - except AttributeError: + match = re.match(r"^label\s+'?\"?([a-zA-Z0-9_-]+)'?\"?$", label) + if match is None: self.warned.append( ( "main_nf", @@ -641,21 +673,46 @@ def check_process_labels(self, lines): ) ) continue - if label not in correct_process_labels: + label = match.group(1) + if label not in correct_process_labels and label not in legacy_process_labels: bad_labels.append(label) + if label in legacy_process_labels: + legacy_labels.append(label) else: good_labels.append(label) - if len(good_labels) > 1: + if legacy_labels: self.warned.append( ( "main_nf", "process_standard_label", - f"Conflicting process labels found: `{'`,`'.join(good_labels)}`", + f"Deprecated process label found: `{legacy_labels[0]}`. Use the new standard labels instead: https://nf-co.re/docs/developing/migration-guides/resource-labels", + self.main_nf, + ) + ) + axes = [label.split("_")[1] for label in good_labels if len(label.split("_")) > 1] + if len(axes) != len(set(axes)): + conflicting = [ + label + for label in good_labels + if axes.count(label.split("_")[1] if len(label.split("_")) > 1 else "") > 1 + ] + self.warned.append( + ( + "main_nf", + "process_standard_label", + f"Conflicting process labels found: `{'`,`'.join(conflicting)}`", + self.main_nf, + ) + ) + elif good_labels: + self.passed.append( + ( + "main_nf", + "process_standard_label", + f"Correct process labels: `{'`,`'.join(good_labels)}`", self.main_nf, ) ) - elif len(good_labels) == 1: - self.passed.append(("main_nf", "process_standard_label", "Correct process label", self.main_nf)) else: self.warned.append(("main_nf", "process_standard_label", "Standard process label not found", self.main_nf)) if len(bad_labels) > 0: @@ -676,6 +733,7 @@ def check_process_labels(self, lines): self.main_nf, ) ) + else: self.warned.append(("main_nf", "process_standard_label", "Process label not specified", self.main_nf)) diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 4d9519d8fd..6ff7c2f87f 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -26,6 +26,49 @@ process { // adding in your local modules too. // TODO nf-core: Customise requirements for specific processes. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors + + // + // Axis-decomposed labels (preferred). Stack one cpus_*, one mem_*, and + // one time_* label on a process to express its resource shape + // independently along each axis. e.g. a cpu-bound, memory-light, fast + // tool can use `process_cpus_high` + `process_mem_low` + `process_time_short`. + // + withLabel:process_cpus_single { + cpus = { 1 } + } + withLabel:process_cpus_low { + cpus = { 2 * task.attempt } + } + withLabel:process_cpus_medium { + cpus = { 6 * task.attempt } + } + withLabel:process_cpus_high { + cpus = { 12 * task.attempt } + } + withLabel:process_mem_low { + memory = { 1.GB * task.attempt } + } + withLabel:process_mem_medium { + memory = { 12.GB * task.attempt } + } + withLabel:process_mem_high { + memory = { 72.GB * task.attempt } + } + withLabel:process_time_short { + time = { 1.h * task.attempt } + } + withLabel:process_time_medium { + time = { 8.h * task.attempt } + } + withLabel:process_time_long { + time = { 20.h * task.attempt } + } + + // + // Legacy combined labels. Kept for backwards compatibility while the + // ecosystem migrates to the axis-decomposed labels above; these will be + // deprecated in a future release. + // withLabel:process_single { cpus = { 1 } memory = { 6.GB * task.attempt } diff --git a/tests/modules/lint/test_main_nf.py b/tests/modules/lint/test_main_nf.py index f6c3848f71..ec5c3229a8 100644 --- a/tests/modules/lint/test_main_nf.py +++ b/tests/modules/lint/test_main_nf.py @@ -17,19 +17,25 @@ @pytest.mark.parametrize( "content,passed,warned,failed", [ - # Valid process label - ("label 'process_high'\ncpus 12", 1, 0, 0), - # Non-alphanumeric characters in label + # Valid new-style axis-decomposed label + ("label 'process_cpus_high'\ncpus 12", 1, 0, 0), + # Multiple axis-decomposed labels on different axes — allowed + ("label 'process_cpus_high'\nlabel 'process_mem_low'\ncpus 12", 1, 0, 0), + # Same axis twice — conflict + ("label 'process_cpus_high'\nlabel 'process_cpus_low'\ncpus 12", 0, 1, 0), + # Legacy label warns (no new-style label present, so also warns "not found") + ("label 'process_high'\ncpus 12", 0, 2, 0), + # Two legacy labels warn legacy + not-found (distinct, so no dup warn) + ("label 'process_high'\nlabel 'process_low'\ncpus 12", 0, 2, 0), + # Duplicate legacy labels: not-found + legacy + duplicate + ("label 'process_high'\nlabel 'process_high'\ncpus 12", 0, 3, 0), + # Non-alphanumeric characters in label: non-alphanumeric warn + not-found warn ("label 'a:label:with:colons'\ncpus 12", 0, 2, 0), - # Conflicting labels - ("label 'process_high'\nlabel 'process_low'\ncpus 12", 0, 1, 0), - # Duplicate labels - ("label 'process_high'\nlabel 'process_high'\ncpus 12", 0, 2, 0), - # Valid and non-standard labels - ("label 'process_high'\nlabel 'process_extra_label'\ncpus 12", 1, 1, 0), - # Non-standard label only - ("label 'process_extra_label'\ncpus 12", 0, 2, 0), - # Non-standard duplicates without quotes + # Non-standard (bad) label only: passes axis check but warns non-standard + ("label 'process_extra_label'\ncpus 12", 1, 1, 0), + # Legacy + non-standard: passes axis check, warns legacy and non-standard + ("label 'process_high'\nlabel 'process_extra_label'\ncpus 12", 1, 2, 0), + # Non-standard duplicates without quotes: conflict + non-standard + duplicate ("label process_extra_label\nlabel process_extra_label\ncpus 12", 0, 3, 0), # No label found ("cpus 12", 0, 1, 0), @@ -158,7 +164,11 @@ def test_topics_and_emits_version_check(self): module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) module_lint.lint(print_results=False, module="bamstats/generalstats") assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" - assert len(module_lint.warned) == 0, f"Expected 0 warnings, got {[x.__dict__ for x in module_lint.warned]}" + # TODO: once modules are migrated to axis-decomposed labels, remove the filter and assert len(module_lint.warned) == 0 + non_label_warned = [w for w in module_lint.warned if w.lint_test != "process_standard_label"] + assert len(non_label_warned) == 0, ( + f"Expected 0 non-label warnings, got {[x.__dict__ for x in non_label_warned]}" + ) assert len(module_lint.passed) > 0