From efb456594f71542dbc5a010cd2ca54a04b544aa6 Mon Sep 17 00:00:00 2001 From: Shen-YuFei Date: Sun, 3 May 2026 16:42:50 +0800 Subject: [PATCH 1/8] chore(pridepy): bump to 0.0.15 --- modules/bigbio/pridepy/environment.yml | 2 +- modules/bigbio/pridepy/main.nf | 6 +++--- modules/bigbio/pridepy/tests/main.nf.test.snap | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/bigbio/pridepy/environment.yml b/modules/bigbio/pridepy/environment.yml index 6fded8f..7a4c533 100644 --- a/modules/bigbio/pridepy/environment.yml +++ b/modules/bigbio/pridepy/environment.yml @@ -3,4 +3,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::pridepy=0.0.14 + - bioconda::pridepy=0.0.15 diff --git a/modules/bigbio/pridepy/main.nf b/modules/bigbio/pridepy/main.nf index 57a54ea..b6de6c0 100644 --- a/modules/bigbio/pridepy/main.nf +++ b/modules/bigbio/pridepy/main.nf @@ -5,8 +5,8 @@ process PRIDEPY_DOWNLOAD { conda "${moduleDir}/environment.yml" container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container - ? 'https://depot.galaxyproject.org/singularity/pridepy:0.0.14--pyhdfd78af_0' - : 'biocontainers/pridepy:0.0.14--pyhdfd78af_0'}" + ? 'https://depot.galaxyproject.org/singularity/pridepy:0.0.15--pyhdfd78af_0' + : 'biocontainers/pridepy:0.0.15--pyhdfd78af_0'}" input: val(meta) @@ -39,7 +39,7 @@ process PRIDEPY_DOWNLOAD { cat <<-END_VERSIONS > versions.yml "${task.process}": - pridepy: 0.0.14 + pridepy: 0.0.15 END_VERSIONS """ } diff --git a/modules/bigbio/pridepy/tests/main.nf.test.snap b/modules/bigbio/pridepy/tests/main.nf.test.snap index c591913..7d0eede 100644 --- a/modules/bigbio/pridepy/tests/main.nf.test.snap +++ b/modules/bigbio/pridepy/tests/main.nf.test.snap @@ -10,7 +10,7 @@ ] ], [ - "versions.yml:md5,c27ccb04c91f92f188b7f4cf2d54ed49" + "versions.yml:md5,19782821de060ab1588d5d4274bdacf0" ] ], "timestamp": "2026-04-28T18:15:37.876014654", @@ -38,7 +38,7 @@ ] ], [ - "versions.yml:md5,c27ccb04c91f92f188b7f4cf2d54ed49" + "versions.yml:md5,19782821de060ab1588d5d4274bdacf0" ] ], "timestamp": "2026-04-28T18:15:43.996367071", From 3ed3d27067163726b4b6b0eba55b3cfbc5940207 Mon Sep 17 00:00:00 2001 From: Shen-YuFei Date: Tue, 5 May 2026 18:49:30 +0800 Subject: [PATCH 2/8] feat(qpx): add QPX Parquet + MuData export module --- modules/bigbio/qpx/environment.yml | 7 ++ modules/bigbio/qpx/main.nf | 78 ++++++++++++++++++++++ modules/bigbio/qpx/meta.yml | 65 ++++++++++++++++++ modules/bigbio/qpx/tests/main.nf.test | 38 +++++++++++ modules/bigbio/qpx/tests/main.nf.test.snap | 17 +++++ modules/bigbio/qpx/tests/nextflow.config | 10 +++ 6 files changed, 215 insertions(+) create mode 100644 modules/bigbio/qpx/environment.yml create mode 100644 modules/bigbio/qpx/main.nf create mode 100644 modules/bigbio/qpx/meta.yml create mode 100644 modules/bigbio/qpx/tests/main.nf.test create mode 100644 modules/bigbio/qpx/tests/main.nf.test.snap create mode 100644 modules/bigbio/qpx/tests/nextflow.config diff --git a/modules/bigbio/qpx/environment.yml b/modules/bigbio/qpx/environment.yml new file mode 100644 index 0000000..f102abe --- /dev/null +++ b/modules/bigbio/qpx/environment.yml @@ -0,0 +1,7 @@ +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - pip: + - qpx diff --git a/modules/bigbio/qpx/main.nf b/modules/bigbio/qpx/main.nf new file mode 100644 index 0000000..123b2a4 --- /dev/null +++ b/modules/bigbio/qpx/main.nf @@ -0,0 +1,78 @@ +process QPX_EXPORT { + tag "qpx_export" + label 'process_medium' + label 'error_retry' + + container "ghcr.io/bigbio/qpx:${params.qpx_version}" + + input: + path(diann_report) + path(pg_matrix) + path(sdrf) + path(diann_log) + val(project_accession) + + output: + path "qpx_output/" , emit: qpx_dataset + path "*.h5mu" , emit: mudata + path "versions.yml", emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = project_accession ?: 'diann' + def pg_arg = pg_matrix ? "--pg-matrix-path ${pg_matrix}" : '' + def log_arg = diann_log ? "--diann-log ${diann_log}" : '' + def acc_arg = project_accession ? "--project-accession ${project_accession}" : '' + def qvalue = params.matrix_qvalue ?: 0.05 + """ + set -o pipefail + qpxc convert diann \\ + --report-path ${diann_report} \\ + --sdrf-file ${sdrf} \\ + ${pg_arg} \\ + ${log_arg} \\ + ${acc_arg} \\ + --output-folder qpx_output \\ + --output-prefix ${prefix} \\ + --qvalue-threshold ${qvalue} \\ + --standardized-intensities \\ + --duckdb-threads ${task.cpus} \\ + --duckdb-max-memory ${task.memory.toGiga()}GB \\ + --compression zstd \\ + ${args} + + python - <<'PY' +from qpx.dataset import Dataset +from qpx.mudata import build_mudata + +ds = Dataset("qpx_output") +mdata = build_mudata(ds) +mdata.write("${prefix}.h5mu") +ds.close() +print(f"MuData: {mdata.n_obs} obs x {mdata.n_vars} vars -> ${prefix}.h5mu") +PY + +cat <<-END_VERSIONS > versions.yml +"${task.process}": + qpx: \$(qpxc --version 2>&1 | sed 's/^qpx //') + mudata: \$(python -c 'import mudata; print(mudata.__version__)') +END_VERSIONS + """ + + stub: + def prefix = project_accession ?: 'diann' + """ + mkdir -p qpx_output + touch qpx_output/stub.parquet + touch ${prefix}.h5mu + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + qpx: stub + mudata: stub + END_VERSIONS + """ +} diff --git a/modules/bigbio/qpx/meta.yml b/modules/bigbio/qpx/meta.yml new file mode 100644 index 0000000..837f3b8 --- /dev/null +++ b/modules/bigbio/qpx/meta.yml @@ -0,0 +1,65 @@ +name: qpx_export +description: Convert DIA-NN quantification outputs to QPX Parquet dataset and MuData (.h5mu) +keywords: + - proteomics + - quantification + - parquet + - mudata + - dia-nn + - qpx +tools: + - qpx: + description: | + QPX (Quantitative Proteomics eXchange) converts tool-specific quantification + outputs into a standardised Parquet dataset and optionally a MuData (.h5mu) file + for downstream analysis in Python (scanpy / muon). + homepage: https://github.com/bigbio/qpx + documentation: https://github.com/bigbio/qpx + tool_dev_url: https://github.com/bigbio/qpx + licence: + - "Apache-2.0" + identifier: "" +input: + - diann_report: + type: file + description: DIA-NN main report (TSV or Parquet) + pattern: "*.{tsv,parquet}" + - pg_matrix: + type: file + description: DIA-NN protein-group matrix + pattern: "*.{tsv,parquet}" + - sdrf: + type: file + description: SDRF sample metadata file + pattern: "*.sdrf.tsv" + - diann_log: + type: file + description: DIA-NN summary log for version detection + pattern: "*.log" + - project_accession: + type: string + description: PRIDE project accession (e.g. PXD026600) used as output prefix +output: + qpx_dataset: + - "qpx_output/": + type: directory + description: QPX Parquet dataset directory + pattern: "qpx_output/" + mudata: + - "*.h5mu": + type: file + description: MuData file containing the quantification data + pattern: "*.h5mu" + versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" + ontologies: + - edam: http://edamontology.org/format_3750 +authors: + - "@ypriverol" + - "@Shen-YuFei" +maintainers: + - "@ypriverol" + - "@Shen-YuFei" diff --git a/modules/bigbio/qpx/tests/main.nf.test b/modules/bigbio/qpx/tests/main.nf.test new file mode 100644 index 0000000..9cc60e0 --- /dev/null +++ b/modules/bigbio/qpx/tests/main.nf.test @@ -0,0 +1,38 @@ +nextflow_process { + + name "Test Process QPX_EXPORT" + script "../main.nf" + process "QPX_EXPORT" + tag "modules" + tag "modules_bigbio" + tag "modules_qpx" + tag "qpx" + + test("Should run stub mode") { + + options "-stub" + config "./nextflow.config" + + when { + process { + """ + input[0] = file("stub_report.tsv", checkIfExists: false) + input[1] = file("stub_pg_matrix.tsv", checkIfExists: false) + input[2] = file("stub_sample.sdrf.tsv", checkIfExists: false) + input[3] = file("stub_diann.log", checkIfExists: false) + input[4] = "PXD026600" + """ + } + } + + then { + assert process.success + assert process.out.qpx_dataset.size() == 1 + assert process.out.mudata.size() == 1 + assert snapshot( + process.out.mudata, + process.out.versions + ).match("stub") + } + } +} diff --git a/modules/bigbio/qpx/tests/main.nf.test.snap b/modules/bigbio/qpx/tests/main.nf.test.snap new file mode 100644 index 0000000..87c0696 --- /dev/null +++ b/modules/bigbio/qpx/tests/main.nf.test.snap @@ -0,0 +1,17 @@ +{ + "stub": { + "content": [ + [ + "PXD026600.h5mu:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "versions.yml:md5,cc457dc644e23b74317eaa02eedda495" + ] + ], + "timestamp": "2026-05-05T18:36:46.564300378", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + } +} \ No newline at end of file diff --git a/modules/bigbio/qpx/tests/nextflow.config b/modules/bigbio/qpx/tests/nextflow.config new file mode 100644 index 0000000..c1cb05c --- /dev/null +++ b/modules/bigbio/qpx/tests/nextflow.config @@ -0,0 +1,10 @@ +process { + withName: 'QPX_EXPORT' { + ext.args = '' + } +} + +params { + qpx_version = 'dev' + matrix_qvalue = 0.05 +} From 05af81180891d142d8b27f553ca947bc83412b20 Mon Sep 17 00:00:00 2001 From: Shen-YuFei Date: Wed, 6 May 2026 20:45:02 +0800 Subject: [PATCH 3/8] test(qpx): add real-data test using PXD063291 DIA-NN results --- modules/bigbio/qpx/tests/main.nf.test | 52 ++++++++++++++++++++++++++- tests/config/test_data.config | 4 +++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/modules/bigbio/qpx/tests/main.nf.test b/modules/bigbio/qpx/tests/main.nf.test index 9cc60e0..84b839a 100644 --- a/modules/bigbio/qpx/tests/main.nf.test +++ b/modules/bigbio/qpx/tests/main.nf.test @@ -8,10 +8,60 @@ nextflow_process { tag "modules_qpx" tag "qpx" + config "./nextflow.config" + + test("Should convert DIA-NN output to QPX Parquet + MuData") { + + when { + def zipUrl = (params.test_data?.proteomics?.qpx?.diann_results) ?: "https://ftp.pride.ebi.ac.uk/pub/databases/pride/resources/proteomes/pmultiqc/example-projects/PXD063291.zip" + def zipFile = file("test_qpx.zip") + zipFile.withOutputStream { out -> + out << new URL(zipUrl).openStream() + } + def resultsDir = file("test_qpx_data") + resultsDir.mkdirs() + new java.util.zip.ZipFile(zipFile).withCloseable { zip -> + zip.entries().each { entry -> + if (!entry.isDirectory()) { + def extractFile = new File(resultsDir, entry.name) + extractFile.parentFile.mkdirs() + extractFile.withOutputStream { out -> + out << zip.getInputStream(entry) + } + } + } + } + zipFile.delete() + + process { + """ + input[0] = file("${resultsDir}/PXD063291/DiaNN/report.tsv") + input[1] = file("${resultsDir}/PXD063291/DiaNN/report.pg_matrix.tsv") + input[2] = file("${resultsDir}/PXD063291/PXD063291.sdrf.tsv") + input[3] = file("${resultsDir}/PXD063291/DiaNN/report.log.txt") + input[4] = "PXD063291" + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.qpx_dataset.size() == 1 }, + { assert process.out.mudata.size() == 1 }, + { assert process.out.mudata[0].toString().endsWith('.h5mu') }, + { assert snapshot(process.out.versions).match("versions") } + ) + } + + cleanup { + file("test_qpx_data").deleteDir() + } + } + test("Should run stub mode") { options "-stub" - config "./nextflow.config" when { process { diff --git a/tests/config/test_data.config b/tests/config/test_data.config index b5dbcf0..c192bc3 100644 --- a/tests/config/test_data.config +++ b/tests/config/test_data.config @@ -23,6 +23,10 @@ params { mzml = "https://ftp.pride.ebi.ac.uk/pub/databases/pride/resources/proteomes/quantms-ci-github/onsite/1.mzML" idxml = "https://ftp.pride.ebi.ac.uk/pub/databases/pride/resources/proteomes/quantms-ci-github/onsite/1_consensus_fdr_filter_pep.idXML" } + qpx { + // DIA-NN results for QPX export testing (reuses pmultiqc DIANN example) + diann_results = "https://ftp.pride.ebi.ac.uk/pub/databases/pride/resources/proteomes/pmultiqc/example-projects/PXD063291.zip" + } } } } From 46525f07ae926012e73afc77b0c2ebc38dc22a8f Mon Sep 17 00:00:00 2001 From: Shen-YuFei Date: Wed, 6 May 2026 20:49:25 +0800 Subject: [PATCH 4/8] fix(qpx): handle null task.memory with fallback --- modules/bigbio/qpx/main.nf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/bigbio/qpx/main.nf b/modules/bigbio/qpx/main.nf index 123b2a4..2611ee2 100644 --- a/modules/bigbio/qpx/main.nf +++ b/modules/bigbio/qpx/main.nf @@ -3,7 +3,10 @@ process QPX_EXPORT { label 'process_medium' label 'error_retry' - container "ghcr.io/bigbio/qpx:${params.qpx_version}" + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://depot.galaxyproject.org/singularity/qpx:' + : 'biocontainers/qpx:'}" input: path(diann_report) @@ -40,7 +43,7 @@ process QPX_EXPORT { --qvalue-threshold ${qvalue} \\ --standardized-intensities \\ --duckdb-threads ${task.cpus} \\ - --duckdb-max-memory ${task.memory.toGiga()}GB \\ + --duckdb-max-memory ${task.memory ? task.memory.toGiga() : 4}GB \\ --compression zstd \\ ${args} From fdd30d5ccd274c5a6039c8e74981abebbadd38fe Mon Sep 17 00:00:00 2001 From: Shen-YuFei Date: Wed, 6 May 2026 20:51:48 +0800 Subject: [PATCH 5/8] fix(qpx): use ghcr.io/bigbio/qpx:dev until BioContainers image is published --- modules/bigbio/qpx/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/bigbio/qpx/main.nf b/modules/bigbio/qpx/main.nf index 2611ee2..6bfe9c8 100644 --- a/modules/bigbio/qpx/main.nf +++ b/modules/bigbio/qpx/main.nf @@ -5,8 +5,8 @@ process QPX_EXPORT { conda "${moduleDir}/environment.yml" container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container - ? 'https://depot.galaxyproject.org/singularity/qpx:' - : 'biocontainers/qpx:'}" + ? 'https://depot.galaxyproject.org/singularity/qpx:1.0.2' + : 'ghcr.io/bigbio/qpx:dev'}" input: path(diann_report) From 7a46c93911d3970dceb21f2e15a2bd684615fa7d Mon Sep 17 00:00:00 2001 From: Shen-YuFei Date: Wed, 6 May 2026 20:54:11 +0800 Subject: [PATCH 6/8] fix(qpx): use absolute paths for test input files --- modules/bigbio/qpx/tests/main.nf.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/bigbio/qpx/tests/main.nf.test b/modules/bigbio/qpx/tests/main.nf.test index 84b839a..3db72db 100644 --- a/modules/bigbio/qpx/tests/main.nf.test +++ b/modules/bigbio/qpx/tests/main.nf.test @@ -35,10 +35,10 @@ nextflow_process { process { """ - input[0] = file("${resultsDir}/PXD063291/DiaNN/report.tsv") - input[1] = file("${resultsDir}/PXD063291/DiaNN/report.pg_matrix.tsv") - input[2] = file("${resultsDir}/PXD063291/PXD063291.sdrf.tsv") - input[3] = file("${resultsDir}/PXD063291/DiaNN/report.log.txt") + input[0] = file('${resultsDir.getAbsolutePath()}/PXD063291/DiaNN/report.tsv') + input[1] = file('${resultsDir.getAbsolutePath()}/PXD063291/DiaNN/report.pg_matrix.tsv') + input[2] = file('${resultsDir.getAbsolutePath()}/PXD063291/PXD063291.sdrf.tsv') + input[3] = file('${resultsDir.getAbsolutePath()}/PXD063291/DiaNN/report.log.txt') input[4] = "PXD063291" """ } From cffb20fdc51581b6e4fc6752a49e7b5ccc8f73bd Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 7 May 2026 06:21:46 +0100 Subject: [PATCH 7/8] Update container image for QPX module --- modules/bigbio/qpx/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/bigbio/qpx/main.nf b/modules/bigbio/qpx/main.nf index 6bfe9c8..85898c5 100644 --- a/modules/bigbio/qpx/main.nf +++ b/modules/bigbio/qpx/main.nf @@ -5,8 +5,8 @@ process QPX_EXPORT { conda "${moduleDir}/environment.yml" container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container - ? 'https://depot.galaxyproject.org/singularity/qpx:1.0.2' - : 'ghcr.io/bigbio/qpx:dev'}" + ? 'https://depot.galaxyproject.org/singularity/qpx:1.0.2--pyhdfd78af_0' + : 'biocontainers/qpx:1.0.2--pyhdfd78af_0'}" input: path(diann_report) From 69529f0bf418a82b75c4d29255f9f73f8ed84dbf Mon Sep 17 00:00:00 2001 From: Shen-YuFei Date: Thu, 7 May 2026 14:13:35 +0800 Subject: [PATCH 8/8] fix(qpx): update container tag to 1.0.2--pyhdfd78af_1 --- modules/bigbio/qpx/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/bigbio/qpx/main.nf b/modules/bigbio/qpx/main.nf index 85898c5..726351f 100644 --- a/modules/bigbio/qpx/main.nf +++ b/modules/bigbio/qpx/main.nf @@ -5,8 +5,8 @@ process QPX_EXPORT { conda "${moduleDir}/environment.yml" container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container - ? 'https://depot.galaxyproject.org/singularity/qpx:1.0.2--pyhdfd78af_0' - : 'biocontainers/qpx:1.0.2--pyhdfd78af_0'}" + ? 'https://depot.galaxyproject.org/singularity/qpx:1.0.2--pyhdfd78af_1' + : 'biocontainers/qpx:1.0.2--pyhdfd78af_1'}" input: path(diann_report)