Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

### Linting

- accept axis-decomposed `process_cpus_*`, `process_mem_*` and `process_time_*` labels alongside the legacy combined labels ([#4265](https://github.com/nf-core/tools/pull/4265))
- accept `process_low_memory` as a standard module label ([#4264](https://github.com/nf-core/tools/pull/4264))

### Modules
Expand All @@ -17,6 +18,7 @@

### Template

- add axis-decomposed `process_cpus_{single,low,medium,high}`, `process_mem_{low,medium,high}` and `process_time_{short,medium,long}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#4265](https://github.com/nf-core/tools/pull/4265))
- add `process_low_memory` resource label to `base.config` ([#4264](https://github.com/nf-core/tools/pull/4264))

#### Version updates
Expand Down
12 changes: 12 additions & 0 deletions nf_core/components/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,18 @@ def _get_bioconda_tool(self):

def _get_module_structure_components(self):
process_label_defaults = [
# Axis-decomposed labels (preferred)
"process_cpus_single",
"process_cpus_low",
"process_cpus_medium",
"process_cpus_high",
"process_mem_low",
"process_mem_medium",
"process_mem_high",
"process_time_short",
"process_time_medium",
"process_time_long",
# Legacy combined labels (kept for backwards compatibility, to be deprecated)
"process_single",
"process_low",
"process_medium",
Expand Down
78 changes: 68 additions & 10 deletions nf_core/modules/lint/main_nf.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import yaml
from rich.progress import Progress

import nf_core
import nf_core.utils
from nf_core.components.components_differ import ComponentsDiffer
from nf_core.components.nfcore_component import NFCoreComponent

Expand Down Expand Up @@ -57,7 +57,26 @@ def main_nf(
equal the number of ``emit:`` outputs whose name starts with ``versions``.
A warning is issued if a legacy YAML-based ``versions`` emit is used instead
of a topic output.

* ``process_standard_label``: Process labels should follow the standard format.
A warning is issued if a legacy label is used. Allowed standard labels are:
"process_cpus_single",
"process_cpus_low",
"process_cpus_medium",
"process_cpus_high",
"process_mem_low",
"process_mem_medium",
"process_mem_high",
"process_time_short",
"process_time_medium",
"process_time_long"
Legacy labels are:
"process_single",
"process_low",
"process_medium",
"process_high",
"process_long",
"process_low_memory",
"process_high_memory"
"""

inputs: list[str] = []
Expand Down Expand Up @@ -616,6 +635,19 @@ def check_process_section(self, lines, registry, fix_version, progress_bar):

def check_process_labels(self, lines):
correct_process_labels = [
# Axis-decomposed labels (preferred)
"process_cpus_single",
"process_cpus_low",
"process_cpus_medium",
"process_cpus_high",
"process_mem_low",
"process_mem_medium",
"process_mem_high",
"process_time_short",
"process_time_medium",
"process_time_long",
]
legacy_process_labels = [
"process_single",
"process_low",
"process_medium",
Expand All @@ -627,11 +659,11 @@ def check_process_labels(self, lines):
all_labels = [line.strip() for line in lines if line.lstrip().startswith("label ")]
bad_labels = []
good_labels = []
legacy_labels = []
if len(all_labels) > 0:
for label in all_labels:
try:
label = re.match(r"^label\s+'?\"?([a-zA-Z0-9_-]+)'?\"?$", label).group(1)
except AttributeError:
match = re.match(r"^label\s+'?\"?([a-zA-Z0-9_-]+)'?\"?$", label)
if match is None:
self.warned.append(
(
"main_nf",
Expand All @@ -641,21 +673,46 @@ def check_process_labels(self, lines):
)
)
continue
if label not in correct_process_labels:
label = match.group(1)
if label not in correct_process_labels and label not in legacy_process_labels:
bad_labels.append(label)
if label in legacy_process_labels:
legacy_labels.append(label)
else:
good_labels.append(label)
if len(good_labels) > 1:
if legacy_labels:
self.warned.append(
(
"main_nf",
"process_standard_label",
f"Conflicting process labels found: `{'`,`'.join(good_labels)}`",
f"Deprecated process label found: `{legacy_labels[0]}`. Use the new standard labels instead: https://nf-co.re/docs/developing/migration-guides/resource-labels",
self.main_nf,
)
)
axes = [label.split("_")[1] for label in good_labels if len(label.split("_")) > 1]
if len(axes) != len(set(axes)):
conflicting = [
label
for label in good_labels
if axes.count(label.split("_")[1] if len(label.split("_")) > 1 else "") > 1
]
self.warned.append(
(
"main_nf",
"process_standard_label",
f"Conflicting process labels found: `{'`,`'.join(conflicting)}`",
self.main_nf,
)
)
elif good_labels:
self.passed.append(
(
"main_nf",
"process_standard_label",
f"Correct process labels: `{'`,`'.join(good_labels)}`",
self.main_nf,
)
)
elif len(good_labels) == 1:
self.passed.append(("main_nf", "process_standard_label", "Correct process label", self.main_nf))
else:
self.warned.append(("main_nf", "process_standard_label", "Standard process label not found", self.main_nf))
if len(bad_labels) > 0:
Expand All @@ -676,6 +733,7 @@ def check_process_labels(self, lines):
self.main_nf,
)
)

else:
self.warned.append(("main_nf", "process_standard_label", "Process label not specified", self.main_nf))

Expand Down
43 changes: 43 additions & 0 deletions nf_core/pipeline-template/conf/base.config
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,49 @@ process {
// adding in your local modules too.
// TODO nf-core: Customise requirements for specific processes.
// See https://www.nextflow.io/docs/latest/config.html#config-process-selectors

//
// Axis-decomposed labels (preferred). Stack one cpus_*, one mem_*, and
// one time_* label on a process to express its resource shape
// independently along each axis. e.g. a cpu-bound, memory-light, fast
// tool can use `process_cpus_high` + `process_mem_low` + `process_time_short`.
//
withLabel:process_cpus_single {
cpus = { 1 }
}
withLabel:process_cpus_low {
cpus = { 2 * task.attempt }
}
withLabel:process_cpus_medium {
cpus = { 6 * task.attempt }
}
withLabel:process_cpus_high {
cpus = { 12 * task.attempt }
}
withLabel:process_mem_low {
memory = { 1.GB * task.attempt }
}
withLabel:process_mem_medium {
memory = { 12.GB * task.attempt }
}
withLabel:process_mem_high {
memory = { 72.GB * task.attempt }
}
withLabel:process_time_short {
time = { 1.h * task.attempt }
}
withLabel:process_time_medium {
time = { 8.h * task.attempt }
}
withLabel:process_time_long {
time = { 20.h * task.attempt }
}

//
// Legacy combined labels. Kept for backwards compatibility while the
// ecosystem migrates to the axis-decomposed labels above; these will be
// deprecated in a future release.
//
withLabel:process_single {
cpus = { 1 }
memory = { 6.GB * task.attempt }
Expand Down
36 changes: 23 additions & 13 deletions tests/modules/lint/test_main_nf.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,25 @@
@pytest.mark.parametrize(
"content,passed,warned,failed",
[
# Valid process label
("label 'process_high'\ncpus 12", 1, 0, 0),
# Non-alphanumeric characters in label
# Valid new-style axis-decomposed label
("label 'process_cpus_high'\ncpus 12", 1, 0, 0),
# Multiple axis-decomposed labels on different axes — allowed
("label 'process_cpus_high'\nlabel 'process_mem_low'\ncpus 12", 1, 0, 0),
# Same axis twice — conflict
("label 'process_cpus_high'\nlabel 'process_cpus_low'\ncpus 12", 0, 1, 0),
# Legacy label warns (no new-style label present, so also warns "not found")
("label 'process_high'\ncpus 12", 0, 2, 0),
# Two legacy labels warn legacy + not-found (distinct, so no dup warn)
("label 'process_high'\nlabel 'process_low'\ncpus 12", 0, 2, 0),
# Duplicate legacy labels: not-found + legacy + duplicate
("label 'process_high'\nlabel 'process_high'\ncpus 12", 0, 3, 0),
# Non-alphanumeric characters in label: non-alphanumeric warn + not-found warn
("label 'a:label:with:colons'\ncpus 12", 0, 2, 0),
# Conflicting labels
("label 'process_high'\nlabel 'process_low'\ncpus 12", 0, 1, 0),
# Duplicate labels
("label 'process_high'\nlabel 'process_high'\ncpus 12", 0, 2, 0),
# Valid and non-standard labels
("label 'process_high'\nlabel 'process_extra_label'\ncpus 12", 1, 1, 0),
# Non-standard label only
("label 'process_extra_label'\ncpus 12", 0, 2, 0),
# Non-standard duplicates without quotes
# Non-standard (bad) label only: passes axis check but warns non-standard
("label 'process_extra_label'\ncpus 12", 1, 1, 0),
# Legacy + non-standard: passes axis check, warns legacy and non-standard
("label 'process_high'\nlabel 'process_extra_label'\ncpus 12", 1, 2, 0),
# Non-standard duplicates without quotes: conflict + non-standard + duplicate
("label process_extra_label\nlabel process_extra_label\ncpus 12", 0, 3, 0),
# No label found
("cpus 12", 0, 1, 0),
Expand Down Expand Up @@ -158,7 +164,11 @@ def test_topics_and_emits_version_check(self):
module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir)
module_lint.lint(print_results=False, module="bamstats/generalstats")
assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}"
assert len(module_lint.warned) == 0, f"Expected 0 warnings, got {[x.__dict__ for x in module_lint.warned]}"
# TODO: once modules are migrated to axis-decomposed labels, remove the filter and assert len(module_lint.warned) == 0
non_label_warned = [w for w in module_lint.warned if w.lint_test != "process_standard_label"]
assert len(non_label_warned) == 0, (
f"Expected 0 non-label warnings, got {[x.__dict__ for x in non_label_warned]}"
)

assert len(module_lint.passed) > 0

Expand Down
Loading