From ce689dccef5df07fa4c25bfe3f7a216903e59efc Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 7 May 2026 14:24:52 +0200 Subject: [PATCH 01/13] switch to json format for nextflow inspect --- nf_core/utils.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index d376559e53..d2e6f84bfb 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -428,24 +428,21 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: log.debug("No config cache found") # Call `nextflow config` - result = run_cmd("nextflow", f"config -flat {wf_path}") + result = run_cmd("nextflow", f"config -flat -o json {wf_path}") if result is not None: nfconfig_raw, _ = result - nfconfig = nfconfig_raw.decode("utf-8") - multiline_key_value_pattern = re.compile(r"(^|\n)([^\n=\s]+?)\s*=\s*((?:(?!\n[^\n=]+?\s*=).)*)", re.DOTALL) - - for config_match in multiline_key_value_pattern.finditer(nfconfig): - k = config_match.group(2).strip() - v = config_match.group(3).strip().strip("'\"") - if k and v == "": - config[k] = "null" - log.debug(f"Config key: {k}, value: empty string") - elif k and v: - config[k] = v - log.debug(f"Config key: {k}, value: {v}") - else: - log.debug(f"Couldn't find key=value config pair:\n {config_match.group(0)}") - del config_match + try: + parsed = json.loads(nfconfig_raw.decode("utf-8")) + for k, v in parsed.items(): + if v is None or v == "": + config[k] = "null" + elif isinstance(v, bool): + config[k] = str(v).lower() + else: + config[k] = str(v) + log.debug(f"Config key: {k}, value: {config[k]}") + except json.JSONDecodeError as e: + log.warning(f"Unable to parse nextflow config output as JSON: {e}") # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. From 32f2e2aaf1934374ac7ada86ef3648e83dce1220 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 7 May 2026 14:25:15 +0200 Subject: [PATCH 02/13] fix pyright warnings in utils.py --- nf_core/utils.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index d2e6f84bfb..f99ba98ce5 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -25,6 +25,7 @@ from typing import TYPE_CHECKING, Any, Literal import git +import git.exc import prompt_toolkit.styles import questionary import requests.auth @@ -119,7 +120,7 @@ def unquote(s: str) -> str: Returns: String with outer quotes removed if present, otherwise original string """ - import ruamel.yaml + import ruamel.yaml.scalarstring if isinstance(s, ruamel.yaml.scalarstring.DoubleQuotedScalarString): return s @@ -446,8 +447,9 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. + + main_nf = Path(wf_path, "main.nf") try: - main_nf = Path(wf_path, "main.nf") with open(main_nf, "rb") as fh: for line in fh: line_str = line.decode("utf-8") @@ -633,7 +635,7 @@ def lazy_init(self) -> None: """ log.debug("Initialising GitHub API requests session") cache_config = setup_requests_cachedir() - super().__init__(**cache_config) + super().__init__(**cache_config) # type: ignore[arg-type] self.setup_github_auth() self.has_init = True @@ -722,13 +724,13 @@ def safe_get(self, url): return request - def get(self, url, **kwargs): + def get(self, url, params=None, **kwargs): """ Initialise the session if we haven't already, then call the superclass get method. """ if not self.has_init: self.lazy_init() - return super().get(url, **kwargs) + return super().get(url, params=params, **kwargs) def request_retry(self, url, post_data=None): """ @@ -1112,17 +1114,12 @@ class SingularityCacheFilePathValidator(questionary.Validator): Validator for file path specified as --singularity-cache-index argument in nf-core pipelines download """ - def validate(self, value): - if len(value.text): - if Path(value.text).is_file(): - return True - else: - raise questionary.ValidationError( - message="Invalid remote cache index file", - cursor_position=len(value.text), - ) - else: - return True + def validate(self, document) -> None: + if len(document.text) and not Path(document.text).is_file(): + raise questionary.ValidationError( + message="Invalid remote cache index file", + cursor_position=len(document.text), + ) def get_repo_releases_branches(pipeline, wfs): From 33ed274efb268d809056f5dccda7be094d6acecb Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 7 May 2026 15:57:53 +0200 Subject: [PATCH 03/13] use nested keys instead of flattened ones now with the new config parser --- nf_core/components/lint/__init__.py | 2 +- nf_core/modules/modules_json.py | 5 +- nf_core/pipelines/bump_version.py | 10 +- .../pipelines/download/container_fetcher.py | 7 +- nf_core/pipelines/lint/files_exist.py | 9 +- nf_core/pipelines/lint/files_unchanged.py | 29 ++-- nf_core/pipelines/lint/multiqc_config.py | 2 +- nf_core/pipelines/lint/nextflow_config.py | 144 ++++++++---------- nf_core/pipelines/lint/plugin_includes.py | 3 +- nf_core/pipelines/lint/version_consistency.py | 11 +- nf_core/pipelines/rocrate.py | 43 ++---- nf_core/pipelines/schema.py | 34 ++--- nf_core/pipelines/sync.py | 14 +- nf_core/test_datasets/test_datasets_utils.py | 2 +- nf_core/utils.py | 45 +++--- tests/test_utils.py | 8 +- 16 files changed, 168 insertions(+), 200 deletions(-) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index efcdc11d90..1793f30da4 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -158,7 +158,7 @@ def __repr__(self) -> str: def _set_registry(self, registry) -> None: if registry is None: - self.registry = self.config.get("docker.registry", "quay.io") + self.registry = self.config.get("docker", {}).get("registry", "quay.io") else: self.registry = registry log.debug(f"Registry set to {self.registry}") diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 716a40e8ea..c47b4378cf 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -81,8 +81,9 @@ def create(self) -> None: UserWarning: If the creation fails """ pipeline_config = nf_core.utils.fetch_wf_config(self.directory) - pipeline_name = pipeline_config.get("manifest.name", "") - pipeline_url = pipeline_config.get("manifest.homePage", "") + manifest = pipeline_config.get("manifest", {}) + pipeline_name = manifest.get("name", "") + pipeline_url = manifest.get("homePage", "") new_modules_json = ModulesJsonType(name=pipeline_name, homePage=pipeline_url, repos={}) if not self.modules_dir.exists(): diff --git a/nf_core/pipelines/bump_version.py b/nf_core/pipelines/bump_version.py index 604a85801d..ebb38aed99 100644 --- a/nf_core/pipelines/bump_version.py +++ b/nf_core/pipelines/bump_version.py @@ -26,8 +26,8 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: new_version (str): The new version tag for the pipeline. Semantic versioning only. """ - # Collect the old and new version numbers - current_version = pipeline_obj.nf_config.get("manifest.version", "").strip(" '\"") + manifest = pipeline_obj.nf_config.get("manifest", {}) + current_version = manifest.get("version", "") or "" if new_version.startswith("v"): log.warning("Stripping leading 'v' from new version number") new_version = new_version[1:] @@ -99,7 +99,7 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: yaml_key=["report_comment"], ) # nf-test snap files - pipeline_name = pipeline_obj.nf_config.get("manifest.name", "").strip(" '\"") + pipeline_name = manifest.get("name", "") snap_files = [f.relative_to(pipeline_obj.wf_path) for f in Path(pipeline_obj.wf_path).glob("tests/pipeline/*.snap")] for snap_file in snap_files: update_file_version( @@ -169,8 +169,8 @@ def bump_nextflow_version(pipeline_obj: Pipeline, new_version: str) -> None: new_version (str): The new version tag for the required Nextflow version. """ - # Collect the old and new version numbers - strip leading non-numeric characters (>=) - current_version = pipeline_obj.nf_config.get("manifest.nextflowVersion", "").strip(" '\"") + manifest = pipeline_obj.nf_config.get("manifest", {}) + current_version = manifest.get("nextflowVersion", "") or "" current_version = re.sub(r"^[^0-9\.]*", "", current_version) new_version = re.sub(r"^[^0-9\.]*", "", new_version) if not current_version: diff --git a/nf_core/pipelines/download/container_fetcher.py b/nf_core/pipelines/download/container_fetcher.py index bfcda9ed54..3916fb1b9c 100644 --- a/nf_core/pipelines/download/container_fetcher.py +++ b/nf_core/pipelines/download/container_fetcher.py @@ -292,8 +292,11 @@ def gather_config_registries(self, workflow_directory: Path, registry_keys: list config_registries = set() for registry_key in registry_keys: - if registry_key in nf_config: - config_registries.add(nf_config[registry_key]) + parts = registry_key.split(".", 1) + if len(parts) == 2 and parts[0] in nf_config and isinstance(nf_config[parts[0]], dict): + val = nf_config[parts[0]].get(parts[1]) + if val: + config_registries.add(val) return config_registries diff --git a/nf_core/pipelines/lint/files_exist.py b/nf_core/pipelines/lint/files_exist.py index 202906c184..9fb55ccedb 100644 --- a/nf_core/pipelines/lint/files_exist.py +++ b/nf_core/pipelines/lint/files_exist.py @@ -120,11 +120,12 @@ def files_exist(self) -> dict[str, list[str]]: # NB: Should all be files, not directories # List of lists. Passes if any of the files in the sublist are found. #: test autodoc - try: - _, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") - except ValueError: + pipeline_name = self.nf_config.get("manifest", {}).get("name", "") + if "/" in pipeline_name: + _, short_name = pipeline_name.split("/") + else: log.warning("Expected manifest.name to be in the format '/'. Will assume it is ''.") - short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + short_name = pipeline_name files_fail = [ [Path(".gitattributes")], diff --git a/nf_core/pipelines/lint/files_unchanged.py b/nf_core/pipelines/lint/files_unchanged.py index 4f9fe3fcd2..015ae822af 100644 --- a/nf_core/pipelines/lint/files_unchanged.py +++ b/nf_core/pipelines/lint/files_unchanged.py @@ -1,6 +1,5 @@ import filecmp import logging -import re import shutil import tempfile from pathlib import Path @@ -66,21 +65,18 @@ def files_unchanged(self) -> dict[str, list[str] | bool]: could_fix: bool = False # Check that we have the minimum required config - required_pipeline_config = { - "manifest.name", - "manifest.description", - "manifest.contributors", - } - missing_pipeline_config = required_pipeline_config.difference(self.nf_config) + manifest_config = self.nf_config.get("manifest", {}) + missing_pipeline_config = {k for k in ("name", "description", "contributors") if not manifest_config.get(k)} if missing_pipeline_config: - return {"ignored": [f"Required pipeline config not found - {missing_pipeline_config}"]} + missing_keys = {f"manifest.{k}" for k in missing_pipeline_config} + return {"ignored": [f"Required pipeline config not found - {missing_keys}"]} try: - prefix, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + prefix, short_name = manifest_config.get("name", "").split("/") except ValueError: log.warning( "Expected manifest.name to be in the format '/'. Will assume it is and default to repo 'nf-core'" ) - short_name = self.nf_config["manifest.name"].strip("\"'") + short_name = manifest_config.get("name", "") prefix = "nf-core" # NB: Should all be files, not directories @@ -118,14 +114,15 @@ def files_unchanged(self) -> dict[str, list[str] | bool]: tmp_dir.mkdir(parents=True) # Create a template.yaml file for the pipeline creation - if "manifest.author" in self.nf_config: - names = self.nf_config["manifest.author"].strip("\"'") - if "manifest.contributors" in self.nf_config: - contributors = self.nf_config["manifest.contributors"] - names = ", ".join(re.findall(r"name:'([^']+)'", contributors)) + names = "" + if manifest_config.get("author"): + names = manifest_config.get("author", "") + contributors = manifest_config.get("contributors", []) + if contributors: + names = ", ".join([c.get("name", "") for c in contributors if c.get("name")]) template_yaml = { "name": short_name, - "description": self.nf_config["manifest.description"].strip("\"'"), + "description": manifest_config.get("description", ""), "author": names, "org": prefix, } diff --git a/nf_core/pipelines/lint/multiqc_config.py b/nf_core/pipelines/lint/multiqc_config.py index 254caa925a..2b2c87e5b9 100644 --- a/nf_core/pipelines/lint/multiqc_config.py +++ b/nf_core/pipelines/lint/multiqc_config.py @@ -98,7 +98,7 @@ def multiqc_config(self) -> dict[str, list[str]]: if "report_comment" not in ignore_configs: # Check that the minimum plugins exist and are coming first in the summary - version = self.nf_config.get("manifest.version", "").strip(" '\"") + version = self.nf_config.get("manifest", {}).get("version", "") or "" # Get the org from .nf-core.yml config, defaulting to "nf-core" _, nf_core_yaml_config = load_tools_config(self.wf_path) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index f5913f5521..cb32565af2 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -1,4 +1,3 @@ -import ast import logging import re from pathlib import Path @@ -170,7 +169,7 @@ def nextflow_config(self) -> dict[str, list[str]]: ] # Lint for plugins - config_plugins = ast.literal_eval(self.nf_config.get("plugins", "[]")) + config_plugins = self.nf_config.get("plugins", []) found_plugins = [] for plugin in config_plugins: if "@" not in plugin: @@ -203,39 +202,34 @@ def nextflow_config(self) -> dict[str, list[str]]: # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get("nextflow_config", []) if self.lint_config is not None else [] - for cfs in config_fail: - for cf in cfs: - if cf in ignore_configs: - ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") - break - if cf in self.nf_config: - passed.append(f"Config variable found: {self._wrap_quotes(cf)}") - break - else: - failed.append(f"Config variable not found: {self._wrap_quotes(cfs)}") - for cfs in config_warn: - for cf in cfs: - if cf in ignore_configs: - ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") - break - if cf in self.nf_config: - passed.append(f"Config variable found: {self._wrap_quotes(cf)}") - break - else: - warned.append(f"Config variable not found: {self._wrap_quotes(cfs)}") + def _config_has_key(key: str) -> bool: + section, name = key.split(".", 1) + return self.nf_config.get(section, {}).get(name) is not None + + for configs, not_found in [(config_fail, failed), (config_warn, warned)]: + for cfs in configs: + for cf in cfs: + if cf in ignore_configs: + ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") + break + if _config_has_key(cf): + passed.append(f"Config variable found: {self._wrap_quotes(cf)}") + break + else: + not_found.append(f"Config variable not found: {self._wrap_quotes(cfs)}") for cf in config_fail_ifdefined: if cf in ignore_configs: ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") - break - if cf not in self.nf_config: + continue + if not _config_has_key(cf): passed.append(f"Config variable (correctly) not found: {self._wrap_quotes(cf)}") else: failed.append(f"Config variable (incorrectly) found: {self._wrap_quotes(cf)}") # Check and warn if the process configuration is done with deprecated syntax - + process_config = self.nf_config.get("process", {}) process_with_deprecated_syntax = list( - {match.group(1) for ck in self.nf_config if (match := re.match(r"^(process\.\$.*?)\.+.*$", ck)) is not None} + {f"process.{ck}" for ck in (process_config if isinstance(process_config, dict) else {}) if re.match(r"^\$", ck)} ) for pd in process_with_deprecated_syntax: warned.append(f"Process configuration is done with deprecated_syntax: {pd}") @@ -244,90 +238,75 @@ def nextflow_config(self) -> dict[str, list[str]]: for k in ["timeline.enabled", "report.enabled", "trace.enabled", "dag.enabled"]: if k in ignore_configs: continue - if self.nf_config.get(k) == "true": - passed.append(f"Config ``{k}`` had correct value: ``{self.nf_config.get(k)}``") + section, name = k.split(".") + val = self.nf_config.get(section, {}).get(name) + if val is True: + passed.append(f"Config ``{k}`` had correct value: ``{val}``") else: - failed.append(f"Config ``{k}`` did not have correct value: ``{self.nf_config.get(k)}``") + failed.append(f"Config ``{k}`` did not have correct value: ``{val}``") _, nf_core_yaml_config = load_tools_config(self.wf_path) org_name = "nf-core" if nf_core_yaml_config and getattr(nf_core_yaml_config, "template", None): org_name = getattr(nf_core_yaml_config.template, "org", org_name) or org_name - if "manifest.name" not in ignore_configs: - # Check that the pipeline name starts with nf-core - try: - manifest_name = self.nf_config.get("manifest.name", "").strip("'\"") - if not manifest_name.startswith(f"{org_name}/"): - raise AssertionError - except (AssertionError, IndexError): - failed.append(f"Config ``manifest.name`` did not begin with ``{org_name}/``:\n {manifest_name}") - else: - passed.append(f"Config ``manifest.name`` began with ``{org_name}/``") - - if "manifest.homePage" not in ignore_configs: - # Check that the homePage is set to the GitHub URL - try: - manifest_homepage = self.nf_config.get("manifest.homePage", "").strip("'\"") - if not manifest_homepage.startswith(f"https://github.com/{org_name}/"): - raise AssertionError - except (AssertionError, IndexError): - failed.append( - f"Config variable ``manifest.homePage`` did not begin with https://github.com/{org_name}/:\n {manifest_homepage}" - ) - + manifest = self.nf_config.get("manifest", {}) + for key, expected_prefix in [ + ("name", f"{org_name}/"), + ("homePage", f"https://github.com/{org_name}/"), + ]: + value = manifest.get(key, "") + if value in ignore_configs: + continue + if value.startswith(expected_prefix): + passed.append(f"Config ``{value}`` began with ``{expected_prefix}``") else: - passed.append(f"Config variable ``manifest.homePage`` began with https://github.com/{org_name}/") + failed.append(f"Config ``{value}`` did not begin with ``{expected_prefix}``") - # Check that the DAG filename ends in ``.svg`` - if "dag.file" in self.nf_config: + dag_file = self.nf_config.get("dag", {}).get("file", "") + if dag_file: default_dag_format = ".html" - if self.nf_config["dag.file"].strip("'\"").endswith(default_dag_format): + if dag_file.endswith(default_dag_format): passed.append(f"Config ``dag.file`` ended with ``{default_dag_format}``") else: failed.append(f"Config ``dag.file`` did not end with ``{default_dag_format}``") # Check that the minimum nextflowVersion is set properly - if "manifest.nextflowVersion" in self.nf_config: - if self.nf_config.get("manifest.nextflowVersion", "").strip("\"'").lstrip("!").startswith(">="): + nextflow_version = self.nf_config.get("manifest", {}).get("nextflowVersion", "") + if nextflow_version: + if nextflow_version.lstrip("!").startswith(">="): passed.append("Config variable ``manifest.nextflowVersion`` started with >= or !>=") else: failed.append( - "Config ``manifest.nextflowVersion`` did not start with ``>=`` or ``!>=`` : " - f"``{self.nf_config.get('manifest.nextflowVersion', '')}``".strip("\"'") + f"Config ``manifest.nextflowVersion`` did not start with ``>=`` or ``!>=`` : ``{nextflow_version}``" ) # Check that the pipeline version contains ``dev`` - if not self.release_mode and "manifest.version" in self.nf_config: - if self.nf_config["manifest.version"].strip(" '\"").endswith("dev"): - passed.append(f"Config ``manifest.version`` ends in ``dev``: ``{self.nf_config['manifest.version']}``") + manifest_version = self.nf_config.get("manifest", {}).get("version", "") + if not self.release_mode and manifest_version: + if manifest_version.endswith("dev"): + passed.append(f"Config ``manifest.version`` ends in ``dev``: ``{manifest_version}``") else: - warned.append( - f"Config ``manifest.version`` should end in ``dev``: ``{self.nf_config['manifest.version']}``" - ) - elif "manifest.version" in self.nf_config: - if "dev" in self.nf_config["manifest.version"]: + warned.append(f"Config ``manifest.version`` should end in ``dev``: ``{manifest_version}``") + elif manifest_version: + if "dev" in manifest_version: failed.append( - "Config ``manifest.version`` should not contain ``dev`` for a release: " - f"``{self.nf_config['manifest.version']}``" + f"Config ``manifest.version`` should not contain ``dev`` for a release: ``{manifest_version}``" ) else: - passed.append( - "Config ``manifest.version`` does not contain ``dev`` for release: " - f"``{self.nf_config['manifest.version']}``" - ) + passed.append(f"Config ``manifest.version`` does not contain ``dev`` for release: ``{manifest_version}``") if "custom_config" not in ignore_configs: # Check if custom profile params are set correctly - if self.nf_config.get("params.custom_config_version", "").strip("'") == "master": + if self.nf_config.get("params", {}).get("custom_config_version", "") == "master": passed.append("Config `params.custom_config_version` is set to `master`") else: failed.append("Config `params.custom_config_version` is not set to `master`") custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/{}".format( - self.nf_config.get("params.custom_config_version", "").strip("'") + self.nf_config.get("params", {}).get("custom_config_version", "") ) - if self.nf_config.get("params.custom_config_base", "").strip("'") == custom_config_base: + if self.nf_config.get("params", {}).get("custom_config_base", "") == custom_config_base: passed.append(f"Config `params.custom_config_base` is set to `{custom_config_base}`") else: failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`") @@ -411,18 +390,19 @@ def nextflow_config(self) -> dict[str, list[str]]: schema.get_schema_types() # Get types from schema for param_name in schema.schema_defaults: param = "params." + param_name + param_value = self.nf_config.get("params", {}).get(param_name) if param in ignore_defaults: ignored.append(f"Config default ignored: {param}") - elif param in self.nf_config: + elif param_value is not None: config_default: str | float | int | None = None schema_default: str | float | int | None = None if schema.schema_types[param_name] == "boolean": schema_default = str(schema.schema_defaults[param_name]).lower() - config_default = str(self.nf_config[param]).lower() + config_default = str(param_value).lower() elif schema.schema_types[param_name] == "number": try: schema_default = float(schema.schema_defaults[param_name]) - config_default = float(self.nf_config[param]) + config_default = float(param_value) except ValueError: failed.append( f"Config default value incorrect: `{param}` is set as type `number` in nextflow_schema.json, but is not a number in `nextflow.config`." @@ -430,19 +410,19 @@ def nextflow_config(self) -> dict[str, list[str]]: elif schema.schema_types[param_name] == "integer": try: schema_default = int(schema.schema_defaults[param_name]) - config_default = int(self.nf_config[param]) + config_default = int(param_value) except ValueError: failed.append( f"Config default value incorrect: `{param}` is set as type `integer` in nextflow_schema.json, but is not an integer in `nextflow.config`." ) else: schema_default = str(schema.schema_defaults[param_name]) - config_default = str(self.nf_config[param]) + config_default = str(param_value) if config_default is not None and config_default == schema_default: passed.append(f"Config default value correct: {param}= {schema_default}") else: failed.append( - f"Config default value incorrect: `{param}` is set as {self._wrap_quotes(schema_default)} in `nextflow_schema.json` but is {self._wrap_quotes(self.nf_config[param])} in `nextflow.config`." + f"Config default value incorrect: `{param}` is set as {self._wrap_quotes(schema_default)} in `nextflow_schema.json` but is {self._wrap_quotes(param_value)} in `nextflow.config`." ) else: schema_default = str(schema.schema_defaults[param_name]) diff --git a/nf_core/pipelines/lint/plugin_includes.py b/nf_core/pipelines/lint/plugin_includes.py index b787405169..335d1a5501 100644 --- a/nf_core/pipelines/lint/plugin_includes.py +++ b/nf_core/pipelines/lint/plugin_includes.py @@ -1,4 +1,3 @@ -import ast import logging import re from pathlib import Path @@ -12,7 +11,7 @@ def plugin_includes(self) -> dict[str, list[str]]: When nf-schema is used in an nf-core pipeline, the include statements of the plugin functions have to use nf-schema instead of nf-validation and vice versa """ - config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", "[]"))] + config_plugins = [plugin.split("@")[0] for plugin in self.nf_config.get("plugins", [])] validation_plugin = "nf-validation" if "nf-validation" in config_plugins else "nf-schema" passed: list[str] = [] diff --git a/nf_core/pipelines/lint/version_consistency.py b/nf_core/pipelines/lint/version_consistency.py index c939938a46..bf006682b9 100644 --- a/nf_core/pipelines/lint/version_consistency.py +++ b/nf_core/pipelines/lint/version_consistency.py @@ -32,15 +32,16 @@ def version_consistency(self): # Get the version definitions # Get version from nextflow.config versions = {} - versions["manifest.version"] = self.nf_config.get("manifest.version", "").strip(" '\"") + versions["manifest.version"] = self.nf_config.get("manifest", {}).get("version", "") or "" # Get version from the docker tag - if self.nf_config.get("process.container", "") and ":" not in self.nf_config.get("process.container", ""): - failed.append(f"Docker slug seems not to have a version tag: {self.nf_config.get('process.container', '')}") + process_container = self.nf_config.get("process", {}).get("container", "") or "" + if process_container and ":" not in process_container: + failed.append(f"Docker slug seems not to have a version tag: {process_container}") # Get config container tag (if set; one container per workflow) - if self.nf_config.get("process.container", ""): - versions["process.container"] = self.nf_config.get("process.container", "").strip(" '\"").split(":")[-1] + if process_container: + versions["process.container"] = process_container.split(":")[-1] # Get version from the $GITHUB_REF env var if this is a release if ( diff --git a/nf_core/pipelines/rocrate.py b/nf_core/pipelines/rocrate.py index b4735c2127..0bc0cf8e56 100644 --- a/nf_core/pipelines/rocrate.py +++ b/nf_core/pipelines/rocrate.py @@ -1,7 +1,6 @@ #!/usr/bin/env python """Code to deal with pipeline RO (Research Object) Crates""" -import json import logging import os import re @@ -103,7 +102,7 @@ def create_rocrate(self, json_path: None | Path = None, zip_path: None | Path = """ # Check that the checkout pipeline version is the same as the requested version - if self.version != "" and self.version != self.pipeline_obj.nf_config.get("manifest.version"): + if self.version and self.version != self.pipeline_obj.nf_config.get("manifest", {}).get("version"): # using git checkout to get the requested version log.info(f"Checking out pipeline version {self.version}") if self.pipeline_obj.repo is None: @@ -119,7 +118,7 @@ def create_rocrate(self, json_path: None | Path = None, zip_path: None | Path = except GitCommandError: log.error(f"Could not checkout version {self.version}") sys.exit(1) - self.version = self.pipeline_obj.nf_config.get("manifest.version", "") + self.version = self.pipeline_obj.nf_config.get("manifest", {}).get("version", "") self.make_workflow_rocrate() # Save just the JSON metadata file @@ -161,13 +160,14 @@ def make_workflow_rocrate(self) -> None: # Create the RO Crate object + manifest = self.pipeline_obj.nf_config.get("manifest", {}) self.crate = custom_make_crate( self.pipeline_dir, self.pipeline_dir / "main.nf", - self.pipeline_obj.nf_config.get("manifest.homePage", ""), - self.pipeline_obj.nf_config.get("manifest.name", ""), - self.pipeline_obj.nf_config.get("manifest.version", ""), - self.pipeline_obj.nf_config.get("manifest.nextflowVersion", ""), + manifest.get("homePage", ""), + manifest.get("name", ""), + manifest.get("version", ""), + manifest.get("nextflowVersion", ""), diagram=diagram, ) @@ -268,10 +268,11 @@ def add_main_authors(self, wf_file: rocrate.model.entity.Entity) -> None: """ Add workflow authors to the crate """ + manifest = self.pipeline_obj.nf_config.get("manifest", {}) contributors = [] - if "manifest.contributors" in self.pipeline_obj.nf_config: + if manifest.get("contributors"): contributors = self.parse_manifest_contributors() - if not contributors and "manifest.author" in self.pipeline_obj.nf_config: + if not contributors and manifest.get("author"): if self.pipeline_obj.repo: contributors = self.parse_manifest_authors() else: @@ -346,7 +347,7 @@ def _make_progress_bar(self): def parse_manifest_authors(self) -> list: # parse manifest.author" - authors = [a.strip() for a in self.pipeline_obj.nf_config["manifest.author"].split(",")] + authors = [a.strip() for a in self.pipeline_obj.nf_config.get("manifest", {}).get("author", "").split(",")] # remove duplicates authors = list(set(authors)) log.debug(f"Authors: {authors}") @@ -398,26 +399,14 @@ def parse_manifest_authors(self) -> list: # and return as a list of dictionaries def parse_manifest_contributors(self) -> list: field_names = ["name", "affiliation", "github", "contribution", "orcid", "email"] - # Grab the contributor list and convert to JSON - # TODO: can be removed once we switch to `nextflow config -o json` - contributors_str = self.pipeline_obj.nf_config["manifest.contributors"] - log.debug(f"manifest.contributors: {contributors_str}") - # JSON uses double quotes, not single quotes - contributors_str = contributors_str.replace("'", '"') - for key in field_names: - # All dictionary keys need to be quoted - contributors_str = contributors_str.replace(f"{key}:", f'"{key}":') - # Use curly brackets for dictionaries - contributors_str = contributors_str.replace("], [", "}, {").replace("[[", "[{").replace("]]", "}]") - log.debug(f"manifest.contributors (normalised): {contributors_str}") - try: - contributors = json.loads(contributors_str) - except json.JSONDecodeError as exc: + # Grab the contributor list directly from the nested config (already a list[dict]) + contributors = self.pipeline_obj.nf_config.get("manifest", {}).get("contributors", []) + log.debug(f"manifest.contributors: {contributors}") + if not isinstance(contributors, list): log.error( "Could not parse `manifest.contributors` from nextflow.config. " "Expected a list of maps, for example: [[name: 'First Last', github: 'user']]. " - f"Normalised string passed to JSON parser was: {contributors_str!r}. " - f"JSON decoding error: {exc}" + f"Got: {contributors!r}" ) return [] diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 0c42b718dc..2deac6042e 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -93,16 +93,17 @@ def _update_validation_plugin_from_config(self) -> None: # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": self.defs_notation = "$defs" + validation = conf.get("validation", {}) + help_config = validation.get("help", {}) ignored_params = [ - conf.get("validation.help.shortParameter", "help"), - conf.get("validation.help.fullParameter", "helpFull"), - conf.get("validation.help.showHiddenParameter", "showHidden"), + help_config.get("shortParameter", "help"), + help_config.get("fullParameter", "helpFull"), + help_config.get("showHiddenParameter", "showHidden"), "trace_report_suffix", # report suffix should be ignored by default as it is a Java Date object ] # Help parameter should be ignored by default - ignored_params_config_str = conf.get("validation.defaultIgnoreParams", "") - ignored_params_config = [ - item.strip().strip("'") for item in ignored_params_config_str[1:-1].split(",") - ] # Extract list elements and remove whitespace + ignored_params_config = validation.get("defaultIgnoreParams", []) + if not isinstance(ignored_params_config, list): + ignored_params_config = [] if len(ignored_params_config) > 0: log.debug(f"Ignoring parameters from config: {ignored_params_config}") @@ -758,16 +759,15 @@ def get_wf_params(self): log.debug("Collecting pipeline parameter defaults\n") config = nf_core.utils.fetch_wf_config(Path(self.schema_filename).parent) skipped_params = [] - # Pull out just the params. values - for ckey, cval in config.items(): - if ckey.startswith("params."): - # skip anything that's not a flat variable - if "." in ckey[7:]: - skipped_params.append(ckey) - continue - self.pipeline_params[ckey[7:]] = cval - if ckey.startswith("manifest."): - self.pipeline_manifest[ckey[9:]] = cval + # Pull out just the params values (top-level flat keys only) + for pkey, pval in config.get("params", {}).items(): + if isinstance(pval, dict): + skipped_params.append(f"params.{pkey}") + continue + self.pipeline_params[pkey] = pval + # Pull out manifest values + for mkey, mval in config.get("manifest", {}).items(): + self.pipeline_manifest[mkey] = mval # Log skipped params if len(skipped_params) > 0: log.debug( diff --git a/nf_core/pipelines/sync.py b/nf_core/pipelines/sync.py index d5a167c6ba..0d814fb331 100644 --- a/nf_core/pipelines/sync.py +++ b/nf_core/pipelines/sync.py @@ -81,10 +81,10 @@ def __init__( self.make_pr = make_pr self.gh_pr_returned_data: dict = {} self.required_config_vars = [ - "manifest.name", - "manifest.description", - "manifest.version", - "manifest.contributors", + "name", + "description", + "version", + "contributors", ] self.force_pr = force_pr @@ -232,8 +232,8 @@ def get_wf_config(self) -> None: # Check that we have the required variables for rvar in self.required_config_vars: - if rvar not in self.wf_config: - raise SyncExceptionError(f"Workflow config variable `{rvar}` not found!") + if rvar not in self.wf_config.get("manifest", {}): + raise SyncExceptionError(f"Workflow config variable `manifest.{rvar}` not found!") def checkout_template_branch(self): """ @@ -320,7 +320,7 @@ def make_template_pipeline(self) -> None: from_config_file=True, no_git=True, force=True, - default_branch=self.wf_config.get("manifest.defaultBranch") or "master", + default_branch=self.wf_config.get("manifest", {}).get("defaultBranch") or "master", ) pipeline_create_obj.init_pipeline() diff --git a/nf_core/test_datasets/test_datasets_utils.py b/nf_core/test_datasets/test_datasets_utils.py index 7ed782cba4..61a7f60b30 100644 --- a/nf_core/test_datasets/test_datasets_utils.py +++ b/nf_core/test_datasets/test_datasets_utils.py @@ -204,7 +204,7 @@ def get_or_prompt_branch(maybe_branch: str) -> tuple[str, list[str]]: branch_prefill = MODULES_BRANCH_NAME elif repo_type == "pipeline": wf_config = fetch_wf_config(base_dir) - pipeline_name = wf_config.get("manifest.name", "").split("/")[-1] + pipeline_name = wf_config.get("manifest", {}).get("name", "").split("/")[-1] if pipeline_name in all_branches: branch_prefill = pipeline_name diff --git a/nf_core/utils.py b/nf_core/utils.py index f99ba98ce5..f1ca661070 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -272,10 +272,11 @@ def load_pipeline_config(self) -> bool: Once loaded, set a few convenience reference class attributes """ self.nf_config = fetch_wf_config(self.wf_path) + manifest = self.nf_config.get("manifest", {}) - self.pipeline_prefix, self.pipeline_name = self.nf_config.get("manifest.name", "/").strip("'").split("/") + self.pipeline_prefix, self.pipeline_name = manifest.get("name", "/").split("/") - nextflow_version_match = re.search(r"[0-9\.]+(-edge)?", self.nf_config.get("manifest.nextflowVersion", "")) + nextflow_version_match = re.search(r"[0-9\.]+(-edge)?", manifest.get("nextflowVersion", "") or "") if nextflow_version_match: self.minNextflowVersion = nextflow_version_match.group(0) return True @@ -370,7 +371,7 @@ def check_nextflow_version(minimal_nf_version: tuple[int, int, int, bool], silen return nf_version >= minimal_nf_version -def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: +def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict[str, Any]: """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -384,7 +385,7 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: log.debug(f"Got '{wf_path}' as path") wf_path = Path(wf_path) - config = {} + config: dict[str, Any] = {} cache_fn = None cache_basedir = None cache_path = None @@ -429,19 +430,14 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: log.debug("No config cache found") # Call `nextflow config` - result = run_cmd("nextflow", f"config -flat -o json {wf_path}") + result = run_cmd("nextflow", f"config -o json {wf_path}") if result is not None: nfconfig_raw, _ = result try: parsed = json.loads(nfconfig_raw.decode("utf-8")) - for k, v in parsed.items(): - if v is None or v == "": - config[k] = "null" - elif isinstance(v, bool): - config[k] = str(v).lower() - else: - config[k] = str(v) - log.debug(f"Config key: {k}, value: {config[k]}") + config.update(parsed) + for k in parsed: + log.debug(f"Config section: {k}") except json.JSONDecodeError as e: log.warning(f"Unable to parse nextflow config output as JSON: {e}") @@ -1472,22 +1468,23 @@ def load_tools_config(directory: str | Path = ".") -> tuple[Path | None, NFCoreY template = tools_config.get("template") config_template_keys = template.keys() if template is not None else [] # Get author names from contributors first, then fallback to author - if "manifest.contributors" in wf_config: - contributors = wf_config["manifest.contributors"] - names = re.findall(r"name:'([^']+)'", contributors) + manifest = wf_config.get("manifest", {}) + contributors = manifest.get("contributors", []) + if contributors: + names = [c.get("name", "") for c in contributors if c.get("name")] author_names = ", ".join(names) - elif "manifest.author" in wf_config: - author_names = wf_config["manifest.author"].strip("'\"") + elif manifest.get("author"): + author_names = manifest.get("author", "") else: author_names = None if nf_core_yaml_config.template is None: # The .nf-core.yml file did not contain template information nf_core_yaml_config.template = NFCoreTemplateConfig( org="nf-core", - name=wf_config["manifest.name"].strip("'\"").split("/")[-1], - description=wf_config["manifest.description"].strip("'\""), + name=manifest.get("name", "/").split("/")[-1], + description=manifest.get("description", ""), author=author_names, - version=wf_config["manifest.version"].strip("'\""), + version=manifest.get("version", ""), outdir=str(directory), is_nfcore=True, ) @@ -1495,10 +1492,10 @@ def load_tools_config(directory: str | Path = ".") -> tuple[Path | None, NFCoreY # The .nf-core.yml file contained the old prefix or skip keys nf_core_yaml_config.template = NFCoreTemplateConfig( org=tools_config["template"].get("prefix", tools_config["template"].get("org", "nf-core")), - name=tools_config["template"].get("name", wf_config["manifest.name"].strip("'\"").split("/")[-1]), - description=tools_config["template"].get("description", wf_config["manifest.description"].strip("'\"")), + name=tools_config["template"].get("name", manifest.get("name", "/").split("/")[-1]), + description=tools_config["template"].get("description", manifest.get("description", "")), author=tools_config["template"].get("author", author_names), - version=tools_config["template"].get("version", wf_config["manifest.version"].strip("'\"")), + version=tools_config["template"].get("version", manifest.get("version", "")), outdir=tools_config["template"].get("outdir", str(directory)), skip_features=tools_config["template"].get("skip", tools_config["template"].get("skip_features")), is_nfcore=tools_config["template"].get("prefix", tools_config["template"].get("org")) == "nf-core", diff --git a/tests/test_utils.py b/tests/test_utils.py index fe5ff56d8e..4994958681 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -249,10 +249,10 @@ def test_set_wd_revert_on_raise(self): @mock.patch("nf_core.utils.run_cmd") def test_fetch_wf_config(self, mock_run_cmd): """Test the fetch_wf_config() regular expression to read config params.""" - mock_run_cmd.return_value = (b"params.param1 ? 'a=b' : ''\nparams.param2 = foo", b"mock") - config = nf_core.utils.fetch_wf_config(".", False) - assert len(config.keys()) == 1 - assert "params.param2" in list(config.keys()) + mock_run_cmd.return_value = (b'{"params": {"param2": "foo"}}', b"mock") + config = nf_core.utils.fetch_wf_config(Path(), False) + assert "params" in config + assert config["params"].get("param2") == "foo" @with_temporary_folder def test_get_wf_files(self, tmpdir): From 153084accaac2a57ad56375d93f231eda97b6040 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 7 May 2026 16:54:22 +0200 Subject: [PATCH 04/13] sanitize nexflow's "json output" --- nf_core/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index f1ca661070..9f41406740 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -434,7 +434,9 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict[str, Any]: if result is not None: nfconfig_raw, _ = result try: - parsed = json.loads(nfconfig_raw.decode("utf-8")) + raw_str = nfconfig_raw.decode("utf-8") + json_start = raw_str.find("{") + parsed = json.loads(raw_str[json_start:] if json_start >= 0 else raw_str) config.update(parsed) for k in parsed: log.debug(f"Config section: {k}") From cd50f3f041276f98a57d8018463d516a14c7ebf5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 7 May 2026 17:09:12 +0200 Subject: [PATCH 05/13] add "null" conversion back --- nf_core/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/utils.py b/nf_core/utils.py index 9f41406740..a9d777d67c 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -437,6 +437,8 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict[str, Any]: raw_str = nfconfig_raw.decode("utf-8") json_start = raw_str.find("{") parsed = json.loads(raw_str[json_start:] if json_start >= 0 else raw_str) + if "params" in parsed: + parsed["params"] = {k: "null" if v is None else v for k, v in parsed["params"].items()} config.update(parsed) for k in parsed: log.debug(f"Config section: {k}") From 63af30f8ba0801c5f3b8a4b4ca8e63408f559e1c Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 7 May 2026 17:37:15 +0200 Subject: [PATCH 06/13] no need to guess types anymore --- nf_core/pipelines/schema.py | 44 ++++++++++++++++++------------------- nf_core/utils.py | 2 -- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 2deac6042e..6eec11bc81 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -921,32 +921,30 @@ def build_schema_param(self, p_val): """ Build a pipeline schema dictionary for an param interactively """ + if p_val is None: + return {"type": "string"} + if isinstance(p_val, bool): + return {"type": "boolean", "default": p_val} if p_val else {"type": "boolean"} + if isinstance(p_val, int): + return {"type": "integer", "default": p_val} + if isinstance(p_val, float): + return {"type": "number", "default": p_val} + + # TODO: remove string branch once old text-format config caches are no longer supported p_val = p_val.strip("\"'") - # p_val is always a string as it is parsed from nextflow config this way + if not p_val or p_val == "null": + return {"type": "string"} + if p_val in ("true", "True"): + return {"type": "boolean", "default": True} + if p_val in ("false", "False"): + return {"type": "boolean"} try: - p_val = float(p_val) - if p_val == int(p_val): - p_val = int(p_val) - p_type = "integer" - else: - p_type = "number" + num = float(p_val) + if num == int(num): + return {"type": "integer", "default": int(num)} + return {"type": "number", "default": num} except ValueError: - p_type = "string" - - # Anything can be "null", means that it is not set - if p_val == "null": - p_val = None - - # Booleans - if p_val in ["true", "false", "True", "False"]: - p_val = p_val in ["true", "True"] # Convert to bool - p_type = "boolean" - - # Don't return a default for anything false-y except 0 - if not p_val and not (p_val == 0 and p_val is not False): - return {"type": p_type} - - return {"type": p_type, "default": p_val} + return {"type": "string", "default": p_val} def launch_web_builder(self): """ diff --git a/nf_core/utils.py b/nf_core/utils.py index a9d777d67c..9f41406740 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -437,8 +437,6 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict[str, Any]: raw_str = nfconfig_raw.decode("utf-8") json_start = raw_str.find("{") parsed = json.loads(raw_str[json_start:] if json_start >= 0 else raw_str) - if "params" in parsed: - parsed["params"] = {k: "null" if v is None else v for k, v in parsed["params"].items()} config.update(parsed) for k in parsed: log.debug(f"Config section: {k}") From 5d833b3b689f0f900b556104d6e5f88b5c2c3e8b Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 May 2026 09:49:54 +0200 Subject: [PATCH 07/13] fix params comparison --- nf_core/pipelines/lint/nextflow_config.py | 2 +- nf_core/utils.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index cb32565af2..401e3f2c36 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -204,7 +204,7 @@ def nextflow_config(self) -> dict[str, list[str]]: def _config_has_key(key: str) -> bool: section, name = key.split(".", 1) - return self.nf_config.get(section, {}).get(name) is not None + return name in self.nf_config.get(section, {}) for configs, not_found in [(config_fail, failed), (config_warn, warned)]: for cfs in configs: diff --git a/nf_core/utils.py b/nf_core/utils.py index 9f41406740..eeb50c5947 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -449,11 +449,12 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict[str, Any]: main_nf = Path(wf_path, "main.nf") try: with open(main_nf, "rb") as fh: + params_section = config.setdefault("params", {}) for line in fh: line_str = line.decode("utf-8") - match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line_str) - if match and match.group(1): - config[match.group(1)] = "null" + match = re.match(r"^\s*params\.([a-zA-Z0-9_]+)\s*=(?!=)", line_str) + if match and match.group(1) not in params_section: + params_section[match.group(1)] = None except FileNotFoundError as e: log.debug(f"Could not open {main_nf} to look for parameter declarations - {e}") From e612eea1e3e6c9c54b1391d8a1e539b5d4b1fa37 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 May 2026 09:54:42 +0200 Subject: [PATCH 08/13] fix boolean parsing --- nf_core/pipelines/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 6eec11bc81..24a3330713 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -384,13 +384,13 @@ def validate_config_default_parameter(self, param, schema_param, config_default) return # if default is null, we're good - if config_default == "null": + if config_default is None or config_default == "null": return # Check variable types in nextflow.config - if schema_param["type"] == "string" and str(config_default) in ["false", "true", "''"]: + if schema_param["type"] == "string" and str(config_default).lower() in ["false", "true", "''"]: self.invalid_nextflow_config_default_parameters[param] = f"String should not be set to `{config_default}`" - if schema_param["type"] == "boolean" and str(config_default) not in ["false", "true"]: + if schema_param["type"] == "boolean" and str(config_default).lower() not in ["false", "true"]: self.invalid_nextflow_config_default_parameters[param] = ( f"Booleans should only be true or false, not `{config_default}`" ) From f7dacfb0b722f787ab28ce0ef1c18a553a853add Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 May 2026 10:14:08 +0200 Subject: [PATCH 09/13] nest flattened params in tests --- tests/pipelines/download/test_download.py | 2 +- tests/pipelines/lint/test_files_exist.py | 4 ++-- tests/pipelines/lint/test_multiqc_config.py | 2 +- tests/pipelines/lint/test_nextflow_config.py | 4 ++-- tests/pipelines/lint/test_version_consistency.py | 4 ++-- tests/pipelines/test_bump_version.py | 6 +++--- tests/pipelines/test_rocrate.py | 2 +- tests/pipelines/test_sync.py | 2 +- tests/test_utils.py | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/pipelines/download/test_download.py b/tests/pipelines/download/test_download.py index 40228994cc..8000a4a413 100644 --- a/tests/pipelines/download/test_download.py +++ b/tests/pipelines/download/test_download.py @@ -241,7 +241,7 @@ def test_wf_use_local_configs(self, tmp_path): # Test the function download_obj.wf_use_local_configs("workflow") wf_config = nf_core.utils.fetch_wf_config(Path(test_outdir, "workflow"), cache_config=False) - assert wf_config["params.custom_config_base"] == f"{test_outdir}/workflow/../configs/" + assert wf_config["params"]["custom_config_base"] == f"{test_outdir}/workflow/../configs/" # # Test that `find_container_images` (uses `nextflow inspect`) fetches the correct Docker images diff --git a/tests/pipelines/lint/test_files_exist.py b/tests/pipelines/lint/test_files_exist.py index 6f5604d74f..2cef8aa105 100644 --- a/tests/pipelines/lint/test_files_exist.py +++ b/tests/pipelines/lint/test_files_exist.py @@ -19,7 +19,7 @@ def test_files_exist_missing_config(self): Path(self.new_pipeline, "CHANGELOG.md").unlink() assert self.lint_obj._load() - self.lint_obj.nf_config["manifest.name"] = "nf-core/testpipeline" + self.lint_obj.nf_config["manifest"]["name"] = "nf-core/testpipeline" results = self.lint_obj.files_exist() assert "File not found: `CHANGELOG.md`" in results["failed"] @@ -61,7 +61,7 @@ def test_files_exist_pass_conditional_nfschema(self): f.write(config) assert self.lint_obj._load() - self.lint_obj.nf_config["manifest.schema"] = "nf-core" + self.lint_obj.nf_config["manifest"]["schema"] = "nf-core" results = self.lint_obj.files_exist() assert results["failed"] == [] assert results["ignored"] == [] diff --git a/tests/pipelines/lint/test_multiqc_config.py b/tests/pipelines/lint/test_multiqc_config.py index 5da6e567ec..e3f691508f 100644 --- a/tests/pipelines/lint/test_multiqc_config.py +++ b/tests/pipelines/lint/test_multiqc_config.py @@ -104,7 +104,7 @@ def test_multiqc_config_report_comment_release_fail(self): lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) lint_obj._load() # bump version - lint_obj.nf_config["manifest.version"] = "1.0" + lint_obj.nf_config["manifest"]["version"] = "1.0" result = lint_obj.multiqc_config() # Reset the file with open(self.multiqc_config_yml, "w") as fh: diff --git a/tests/pipelines/lint/test_nextflow_config.py b/tests/pipelines/lint/test_nextflow_config.py index 63c58b149b..6b5cbb36c7 100644 --- a/tests/pipelines/lint/test_nextflow_config.py +++ b/tests/pipelines/lint/test_nextflow_config.py @@ -34,7 +34,7 @@ def test_nextflow_config_bad_name_fail(self): lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) lint_obj.load_pipeline_config() - lint_obj.nf_config["manifest.name"] = "bad_name" + lint_obj.nf_config["manifest"]["name"] = "bad_name" result = lint_obj.nextflow_config() assert len(result["failed"]) > 0 assert len(result["warned"]) == 0 @@ -45,7 +45,7 @@ def test_nextflow_config_dev_in_release_mode_failed(self): lint_obj.load_pipeline_config() lint_obj.release_mode = True - lint_obj.nf_config["manifest.version"] = "dev_is_bad_name" + lint_obj.nf_config["manifest"]["version"] = "dev_is_bad_name" result = lint_obj.nextflow_config() assert len(result["failed"]) > 0 assert len(result["warned"]) == 0 diff --git a/tests/pipelines/lint/test_version_consistency.py b/tests/pipelines/lint/test_version_consistency.py index c7f7f28919..d11b3d3369 100644 --- a/tests/pipelines/lint/test_version_consistency.py +++ b/tests/pipelines/lint/test_version_consistency.py @@ -35,7 +35,7 @@ def test_version_consistency_pass(self): lint_obj.load_pipeline_config() lint_obj.nextflow_config() # Set the version for the container - lint_obj.nf_config["process.container"] = "nfcore/pipeline:1.0.0" + lint_obj.nf_config["process"]["container"] = "nfcore/pipeline:1.0.0" result = lint_obj.version_consistency() assert result["passed"] == [ "Version tags are consistent: manifest.version = 1.0.0, process.container = 1.0.0, nfcore_yml.version = 1.0.0", @@ -73,7 +73,7 @@ def test_version_consistency_not_consistent(self): lint_obj.load_pipeline_config() lint_obj.nextflow_config() - lint_obj.nf_config["process.container"] = "nfcore/pipeline:0.1" + lint_obj.nf_config["process"]["container"] = "nfcore/pipeline:0.1" result = lint_obj.version_consistency() assert len(result["passed"]) == 0 assert result["failed"] == [ diff --git a/tests/pipelines/test_bump_version.py b/tests/pipelines/test_bump_version.py index 3f34c18533..85c78e96a1 100644 --- a/tests/pipelines/test_bump_version.py +++ b/tests/pipelines/test_bump_version.py @@ -24,7 +24,7 @@ def test_bump_pipeline_version(self): # Check nextflow.config new_pipeline_obj.load_pipeline_config() - assert new_pipeline_obj.nf_config["manifest.version"].strip("'\"") == "1.1.0" + assert new_pipeline_obj.nf_config["manifest"]["version"].strip("'\"") == "1.1.0" # Check multiqc_config.yml with open(new_pipeline_obj._fp("assets/multiqc_config.yml")) as fh: @@ -47,7 +47,7 @@ def test_dev_bump_pipeline_version(self): # Check the pipeline config new_pipeline_obj.load_pipeline_config() - assert new_pipeline_obj.nf_config["manifest.version"].strip("'\"") == "1.2dev" + assert new_pipeline_obj.nf_config["manifest"]["version"].strip("'\"") == "1.2dev" def test_bump_nextflow_version(self): # Bump the version number to a specific version, preferably one @@ -58,7 +58,7 @@ def test_bump_nextflow_version(self): new_pipeline_obj._load() # Check nextflow.config - assert new_pipeline_obj.nf_config["manifest.nextflowVersion"].strip("'\"") == f"!>={version}" + assert new_pipeline_obj.nf_config["manifest"]["nextflowVersion"].strip("'\"") == f"!>={version}" # Check .github/workflows/nf-test.yml with open(new_pipeline_obj._fp(".github/workflows/nf-test.yml")) as fh: diff --git a/tests/pipelines/test_rocrate.py b/tests/pipelines/test_rocrate.py index 30df70f821..20ffbafd13 100644 --- a/tests/pipelines/test_rocrate.py +++ b/tests/pipelines/test_rocrate.py @@ -278,7 +278,7 @@ def test_parse_manifest_contributors_logs_parse_errors(self): # Set nf_config directly to avoid running nextflow config -flat on invalid Groovy syntax # (unquoted `alice` is valid Groovy but references an undefined variable, causing nextflow to fail) - self.rocrate_obj.pipeline_obj.nf_config["manifest.contributors"] = ( + self.rocrate_obj.pipeline_obj.nf_config["manifest"]["contributors"] = ( "[[\n name: 'Alice Example',\n github: alice\n]]" ) diff --git a/tests/pipelines/test_sync.py b/tests/pipelines/test_sync.py index aa27170f26..02c1f9bb67 100644 --- a/tests/pipelines/test_sync.py +++ b/tests/pipelines/test_sync.py @@ -148,7 +148,7 @@ def test_get_wf_config_missing_required_config(self): psync.inspect_sync_dir() psync.get_wf_config() # Check that we did actually get some config back - assert psync.wf_config["params.validate_params"] == "true" + assert psync.wf_config["params"]["validate_params"] == "true" # Check that we raised because of the missing fake config var assert exc_info.value.args[0] == "Workflow config variable `fakethisdoesnotexist` not found!" diff --git a/tests/test_utils.py b/tests/test_utils.py index 4994958681..d417c720f7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -104,7 +104,7 @@ def test_rich_force_colours_true(self): def testload_pipeline_config(self): """Load the pipeline Nextflow config""" self.pipeline_obj.load_pipeline_config() - assert self.pipeline_obj.nf_config["dag.enabled"] == "true" + assert self.pipeline_obj.nf_config["dag"]["enabled"] == "true" # TODO nf-core: Assess and strip out if no longer required for DSL2 From 52aa63277766afc92471a1a68a0d8e0009715d78 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 May 2026 11:08:10 +0200 Subject: [PATCH 10/13] fix tests --- nf_core/utils.py | 2 +- tests/pipelines/download/test_singularity.py | 10 +++++----- tests/pipelines/lint/test_nextflow_config.py | 6 +++++- tests/pipelines/test_sync.py | 2 +- tests/test_utils.py | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index eeb50c5947..a76b97342a 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -453,7 +453,7 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict[str, Any]: for line in fh: line_str = line.decode("utf-8") match = re.match(r"^\s*params\.([a-zA-Z0-9_]+)\s*=(?!=)", line_str) - if match and match.group(1) not in params_section: + if match: params_section[match.group(1)] = None except FileNotFoundError as e: diff --git a/tests/pipelines/download/test_singularity.py b/tests/pipelines/download/test_singularity.py index eb6bfe1bbb..78958dbcb6 100644 --- a/tests/pipelines/download/test_singularity.py +++ b/tests/pipelines/download/test_singularity.py @@ -456,11 +456,11 @@ def test_gather_registries_singularity(self, tmp_path, mock_cachedir_prompt, moc container_cache_index=None, ) mock_fetch_wf_config.return_value = { - "apptainer.registry": "apptainer-registry.io", - "docker.registry": "docker.io", - "podman.registry": "podman-registry.io", - "singularity.registry": "singularity-registry.io", - "someother.registry": "fake-registry.io", + "apptainer": {"registry": "apptainer-registry.io"}, + "docker": {"registry": "docker.io"}, + "podman": {"registry": "podman-registry.io"}, + "singularity": {"registry": "singularity-registry.io"}, + "someother": {"registry": "fake-registry.io"}, } singularity_fetcher.registry_set = singularity_fetcher.gather_registries(tmp_path) assert singularity_fetcher.registry_set diff --git a/tests/pipelines/lint/test_nextflow_config.py b/tests/pipelines/lint/test_nextflow_config.py index 6b5cbb36c7..493aa49a76 100644 --- a/tests/pipelines/lint/test_nextflow_config.py +++ b/tests/pipelines/lint/test_nextflow_config.py @@ -105,9 +105,13 @@ def test_catch_params_assignment_in_main_nf(self): lint_obj.load_pipeline_config() result = lint_obj.nextflow_config() assert len(result["failed"]) == 2 + assert ( + result["failed"][0] + == "Config `params.custom_config_base` is not set to `https://raw.githubusercontent.com/nf-core/configs/master`" + ) assert ( result["failed"][1] - == "Config default value incorrect: `params.custom_config_base` is set as `https://raw.githubusercontent.com/nf-core/configs/master` in `nextflow_schema.json` but is `null` in `nextflow.config`." + == "Default value from the Nextflow schema `params.custom_config_base = `https://raw.githubusercontent.com/nf-core/configs/master`` not found in `nextflow.config`." ) def test_allow_params_reference_in_main_nf(self): diff --git a/tests/pipelines/test_sync.py b/tests/pipelines/test_sync.py index 02c1f9bb67..07253f8185 100644 --- a/tests/pipelines/test_sync.py +++ b/tests/pipelines/test_sync.py @@ -148,7 +148,7 @@ def test_get_wf_config_missing_required_config(self): psync.inspect_sync_dir() psync.get_wf_config() # Check that we did actually get some config back - assert psync.wf_config["params"]["validate_params"] == "true" + assert psync.wf_config["params"]["validate_params"] is True # Check that we raised because of the missing fake config var assert exc_info.value.args[0] == "Workflow config variable `fakethisdoesnotexist` not found!" diff --git a/tests/test_utils.py b/tests/test_utils.py index d417c720f7..aaf6e157bd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -104,7 +104,7 @@ def test_rich_force_colours_true(self): def testload_pipeline_config(self): """Load the pipeline Nextflow config""" self.pipeline_obj.load_pipeline_config() - assert self.pipeline_obj.nf_config["dag"]["enabled"] == "true" + assert self.pipeline_obj.nf_config["dag"]["enabled"] is True # TODO nf-core: Assess and strip out if no longer required for DSL2 From 347b03cd54c1364e8245c0f327c68088770aa874 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 May 2026 11:30:24 +0200 Subject: [PATCH 11/13] simplify code --- nf_core/pipelines/lint/files_unchanged.py | 7 +++---- nf_core/pipelines/lint/nextflow_config.py | 11 ++++++----- nf_core/pipelines/schema.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/pipelines/lint/files_unchanged.py b/nf_core/pipelines/lint/files_unchanged.py index 015ae822af..20cef65ae5 100644 --- a/nf_core/pipelines/lint/files_unchanged.py +++ b/nf_core/pipelines/lint/files_unchanged.py @@ -114,12 +114,11 @@ def files_unchanged(self) -> dict[str, list[str] | bool]: tmp_dir.mkdir(parents=True) # Create a template.yaml file for the pipeline creation - names = "" - if manifest_config.get("author"): - names = manifest_config.get("author", "") contributors = manifest_config.get("contributors", []) if contributors: - names = ", ".join([c.get("name", "") for c in contributors if c.get("name")]) + names = ", ".join(c.get("name", "") for c in contributors if c.get("name")) + else: + names = manifest_config.get("author", "") template_yaml = { "name": short_name, "description": manifest_config.get("description", ""), diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 401e3f2c36..caea0fa352 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -255,13 +255,14 @@ def _config_has_key(key: str) -> bool: ("name", f"{org_name}/"), ("homePage", f"https://github.com/{org_name}/"), ]: - value = manifest.get(key, "") - if value in ignore_configs: + config_key = f"manifest.{key}" + if config_key in ignore_configs: continue + value = manifest.get(key, "") if value.startswith(expected_prefix): - passed.append(f"Config ``{value}`` began with ``{expected_prefix}``") + passed.append(f"Config ``{config_key}`` began with ``{expected_prefix}``") else: - failed.append(f"Config ``{value}`` did not begin with ``{expected_prefix}``") + failed.append(f"Config ``{config_key}`` did not begin with ``{expected_prefix}``") dag_file = self.nf_config.get("dag", {}).get("file", "") if dag_file: @@ -272,7 +273,7 @@ def _config_has_key(key: str) -> bool: failed.append(f"Config ``dag.file`` did not end with ``{default_dag_format}``") # Check that the minimum nextflowVersion is set properly - nextflow_version = self.nf_config.get("manifest", {}).get("nextflowVersion", "") + nextflow_version = manifest.get("nextflowVersion", "") if nextflow_version: if nextflow_version.lstrip("!").startswith(">="): passed.append("Config variable ``manifest.nextflowVersion`` started with >= or !>=") diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 24a3330713..eb670d3f05 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -924,7 +924,7 @@ def build_schema_param(self, p_val): if p_val is None: return {"type": "string"} if isinstance(p_val, bool): - return {"type": "boolean", "default": p_val} if p_val else {"type": "boolean"} + return {"type": "boolean", "default": p_val} if p_val else {"type": "boolean"} # no default for False if isinstance(p_val, int): return {"type": "integer", "default": p_val} if isinstance(p_val, float): From c9f76ab5db15911608f9e0283706f516bf6ab7d3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 May 2026 11:33:34 +0200 Subject: [PATCH 12/13] add pytest-clarity and sugar for nicer pytest logs --- pyproject.toml | 2 ++ uv.lock | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a9e78a29e6..a263340cd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,8 @@ dev = [ "types-setuptools", "typing_extensions>=4.0.0", "pytest-asyncio", + "pytest-clarity>=1.0.1", + "pytest-sugar>=1.1.1", "pytest-textual-snapshot==1.1.0", "pytest-workflow>=2.0.0", "pytest-xdist>=3.7.0", diff --git a/uv.lock b/uv.lock index ac0b46a1df..5038d67678 100644 --- a/uv.lock +++ b/uv.lock @@ -1654,8 +1654,10 @@ dev = [ { name = "myst-parser", version = "5.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-clarity" }, { name = "pytest-cov" }, { name = "pytest-datafiles" }, + { name = "pytest-sugar" }, { name = "pytest-textual-snapshot" }, { name = "pytest-workflow" }, { name = "pytest-xdist" }, @@ -1693,8 +1695,10 @@ requires-dist = [ { name = "pygithub" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'" }, + { name = "pytest-clarity", marker = "extra == 'dev'", specifier = ">=1.0.1" }, { name = "pytest-cov", marker = "extra == 'dev'" }, { name = "pytest-datafiles", marker = "extra == 'dev'" }, + { name = "pytest-sugar", marker = "extra == 'dev'", specifier = ">=1.1.1" }, { name = "pytest-textual-snapshot", marker = "extra == 'dev'", specifier = "==1.1.0" }, { name = "pytest-workflow", marker = "extra == 'dev'", specifier = ">=2.0.0" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.7.0" }, @@ -2153,6 +2157,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pprintpp" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/1a/7737e7a0774da3c3824d654993cf57adc915cb04660212f03406334d8c0b/pprintpp-0.4.0.tar.gz", hash = "sha256:ea826108e2c7f49dc6d66c752973c3fc9749142a798d6b254e1e301cfdbc6403", size = 17995, upload-time = "2018-07-01T01:42:34.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/d1/e4ed95fdd3ef13b78630280d9e9e240aeb65cc7c544ec57106149c3942fb/pprintpp-0.4.0-py2.py3-none-any.whl", hash = "sha256:b6b4dcdd0c0c0d75e4d7b2f21a9e933e5b2ce62b26e1a54537f9651ae5a5c01d", size = 16952, upload-time = "2018-07-01T01:42:36.496Z" }, +] + [[package]] name = "prek" version = "0.3.13" @@ -2601,6 +2614,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] +[[package]] +name = "pytest-clarity" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pprintpp" }, + { name = "pytest" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/5c/cafa97944de55738a6a2c5a7cee00d073cb80495032d2b112c4546525eca/pytest-clarity-1.0.1.tar.gz", hash = "sha256:505fe345fad4fe11c6a4187fe683f2c7c52c077caa1e135f3e483fe112db7772", size = 4891, upload-time = "2021-06-11T18:16:18.372Z" } + [[package]] name = "pytest-cov" version = "7.1.0" @@ -2627,6 +2651,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/23/88880d5dc36eac9696ad40823e37c1fdbfa81a67b9cfd0ed445b3ed7ec54/pytest_datafiles-3.0.1-py3-none-any.whl", hash = "sha256:c48f27a8afec03a482a10587ab5f5ab930ced4cbdeb2e3a2cbcc0323a51d22c7", size = 6357, upload-time = "2026-01-04T15:08:25.132Z" }, ] +[[package]] +name = "pytest-sugar" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "termcolor" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/4e/60fed105549297ba1a700e1ea7b828044842ea27d72c898990510b79b0e2/pytest-sugar-1.1.1.tar.gz", hash = "sha256:73b8b65163ebf10f9f671efab9eed3d56f20d2ca68bda83fa64740a92c08f65d", size = 16533, upload-time = "2025-08-23T12:19:35.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/d5/81d38a91c1fdafb6711f053f5a9b92ff788013b19821257c2c38c1e132df/pytest_sugar-1.1.1-py3-none-any.whl", hash = "sha256:2f8319b907548d5b9d03a171515c1d43d2e38e32bd8182a1781eb20b43344cc8", size = 11440, upload-time = "2025-08-23T12:19:34.894Z" }, +] + [[package]] name = "pytest-textual-snapshot" version = "1.1.0" @@ -3334,6 +3371,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, ] +[[package]] +name = "termcolor" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, +] + [[package]] name = "textual" version = "8.2.4" From f8fe5dd232561f241a09021da459c14f1a5283b9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 May 2026 11:42:08 +0200 Subject: [PATCH 13/13] make `required_config_vars` also a nested list --- nf_core/pipelines/sync.py | 18 ++++++++---------- tests/pipelines/test_sync.py | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/nf_core/pipelines/sync.py b/nf_core/pipelines/sync.py index 0d814fb331..003880491d 100644 --- a/nf_core/pipelines/sync.py +++ b/nf_core/pipelines/sync.py @@ -53,7 +53,7 @@ class PipelineSync: original_branch (str): Repo branch that was checked out before we started. made_changes (bool): Whether making the new template pipeline introduced any changes make_pr (bool): Whether to try to automatically make a PR on GitHub.com - required_config_vars (list): List of nextflow variables required to make template pipeline + required_config_vars (dict): Nextflow config variables required to make template pipeline, keyed by section gh_username (str): GitHub username gh_repo (str): GitHub repository name """ @@ -80,12 +80,9 @@ def __init__( self.made_changes = False self.make_pr = make_pr self.gh_pr_returned_data: dict = {} - self.required_config_vars = [ - "name", - "description", - "version", - "contributors", - ] + self.required_config_vars: dict[str, list[str]] = { + "manifest": ["name", "description", "version", "contributors"], + } self.force_pr = force_pr self.gh_username = gh_username @@ -231,9 +228,10 @@ def get_wf_config(self) -> None: self.wf_config = nf_core.utils.fetch_wf_config(Path(self.pipeline_dir)) # Check that we have the required variables - for rvar in self.required_config_vars: - if rvar not in self.wf_config.get("manifest", {}): - raise SyncExceptionError(f"Workflow config variable `manifest.{rvar}` not found!") + for section, keys in self.required_config_vars.items(): + for key in keys: + if key not in self.wf_config.get(section, {}): + raise SyncExceptionError(f"Workflow config variable `{section}.{key}` not found!") def checkout_template_branch(self): """ diff --git a/tests/pipelines/test_sync.py b/tests/pipelines/test_sync.py index 07253f8185..ffcf9bb9c1 100644 --- a/tests/pipelines/test_sync.py +++ b/tests/pipelines/test_sync.py @@ -143,14 +143,14 @@ def test_get_wf_config_missing_required_config(self): """Try getting a workflow config, then make it miss a required config option""" # Try to sync, check we halt with the right error psync = nf_core.pipelines.sync.PipelineSync(self.pipeline_dir) - psync.required_config_vars = ["fakethisdoesnotexist"] + psync.required_config_vars = {"fakesection": ["fakethisdoesnotexist"]} with pytest.raises(nf_core.pipelines.sync.SyncExceptionError) as exc_info: psync.inspect_sync_dir() psync.get_wf_config() # Check that we did actually get some config back assert psync.wf_config["params"]["validate_params"] is True # Check that we raised because of the missing fake config var - assert exc_info.value.args[0] == "Workflow config variable `fakethisdoesnotexist` not found!" + assert exc_info.value.args[0] == "Workflow config variable `fakesection.fakethisdoesnotexist` not found!" def test_checkout_template_branch(self): """Try checking out the TEMPLATE branch of the pipeline"""