From 2973586f27e22d43a3cd8cc19ec3be84ae1c1395 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 10 Mar 2026 13:57:13 -0400 Subject: [PATCH 01/11] Prefix IDs with asap in GeoMx workflow --- workflows/spatial_geomx/main.wdl | 10 +++---- .../spatial_geomx/preprocess/preprocess.wdl | 28 +++++++++---------- workflows/spatial_geomx/structs.wdl | 12 ++++---- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/workflows/spatial_geomx/main.wdl b/workflows/spatial_geomx/main.wdl index 168e0e6..cd88f46 100644 --- a/workflows/spatial_geomx/main.wdl +++ b/workflows/spatial_geomx/main.wdl @@ -54,11 +54,11 @@ workflow spatial_geomx_analysis { call Preprocess.preprocess { input: - team_id = project.team_id, - dataset_id = project.dataset_id, - dataset_doi_url = project.dataset_doi_url, + team_id = project.asap_team_id, + dataset_id = project.asap_dataset_id, + dataset_doi_url = project.asap_dataset_doi_url, slides = project.slides, - project_sample_metadata_csv = project.project_sample_metadata_csv, + project_sample_metadata_csv = project.asap_project_sample_metadata_csv, geomx_config_ini = project.geomx_config_ini, geomxngs_config_pkc = geomxngs_config_pkc, min_segment_reads = min_segment_reads, @@ -115,7 +115,7 @@ workflow spatial_geomx_analysis { if (project.run_project_cohort_analysis) { call CohortAnalysis.cohort_analysis as project_cohort_analysis { input: - cohort_id = project.team_id, + cohort_id = project.asap_team_id, project_sample_ids = preprocess.project_sample_ids, processed_adata_objects = process_to_adata.processed_adata_objects, preprocessing_output_file_paths = preprocessing_output_file_paths, diff --git a/workflows/spatial_geomx/preprocess/preprocess.wdl b/workflows/spatial_geomx/preprocess/preprocess.wdl index 2a129d3..d3050c2 100644 --- a/workflows/spatial_geomx/preprocess/preprocess.wdl +++ b/workflows/spatial_geomx/preprocess/preprocess.wdl @@ -49,9 +49,9 @@ workflow preprocess { String qc_raw_data_path = "~{workflow_raw_data_path_prefix}/qc/~{qc_task_version}" scatter (slide_object in slides) { - String fastq_to_dcc_output = "~{dcc_raw_data_path}/~{slide_object.slide_id}.geomxngs_out_dir.tar.gz" - String dcc_to_rds_output = "~{rds_raw_data_path}/~{slide_object.slide_id}.NanoStringGeoMxSet.rds" - String qc_output = "~{qc_raw_data_path}/~{slide_object.slide_id}.qc.rds" + String fastq_to_dcc_output = "~{dcc_raw_data_path}/~{slide_object.asap_slide_id}.geomxngs_out_dir.tar.gz" + String dcc_to_rds_output = "~{rds_raw_data_path}/~{slide_object.asap_slide_id}.NanoStringGeoMxSet.rds" + String qc_output = "~{qc_raw_data_path}/~{slide_object.asap_slide_id}.qc.rds" } # For each sample, outputs an array of true/false: [fastq_to_dcc_complete, dcc_to_rds_complete, qc_complete] @@ -69,7 +69,7 @@ workflow preprocess { scatter (sample_index in range(length(slide.samples))) { Sample sample = slide.samples[sample_index] - Array[String] project_sample_id = [team_id, sample.sample_id, dataset_doi_url] + Array[String] project_sample_id = [team_id, sample.asap_sample_id, dataset_doi_url] Array[File] fastq_R1s = sample.fastq_R1s Array[File] fastq_R2s = sample.fastq_R2s } @@ -78,13 +78,13 @@ workflow preprocess { String dcc_to_rds_complete = check_output_files_exist.sample_preprocessing_complete[slide_index][1] String qc_complete = check_output_files_exist.sample_preprocessing_complete[slide_index][2] - String fastq_to_dcc_geomxngs_dcc_zip = "~{dcc_raw_data_path}/~{slide.slide_id}.DCC.zip" - String fastq_to_dcc_geomxngs_output_tar_gz = "~{dcc_raw_data_path}/~{slide.slide_id}.geomxngs_out_dir.tar.gz" + String fastq_to_dcc_geomxngs_dcc_zip = "~{dcc_raw_data_path}/~{slide.asap_slide_id}.DCC.zip" + String fastq_to_dcc_geomxngs_output_tar_gz = "~{dcc_raw_data_path}/~{slide.asap_slide_id}.geomxngs_out_dir.tar.gz" if (fastq_to_dcc_complete == "false") { call fastq_to_dcc { input: - slide_id = slide.slide_id, + slide_id = slide.asap_slide_id, fastq_R1s = flatten(fastq_R1s), fastq_R2s = flatten(fastq_R2s), geomx_config_ini = geomx_config_ini, @@ -99,14 +99,14 @@ workflow preprocess { File geomxngs_dcc_zip_output = select_first([fastq_to_dcc.geomxngs_dcc_zip, fastq_to_dcc_geomxngs_dcc_zip]) #!FileCoercion File geomxngs_output_tar_gz_output = select_first([fastq_to_dcc.geomxngs_output_tar_gz, fastq_to_dcc_geomxngs_output_tar_gz]) #!FileCoercion - String dcc_to_rds_object = "~{rds_raw_data_path}/~{slide.slide_id}.NanoStringGeoMxSet.rds" + String dcc_to_rds_object = "~{rds_raw_data_path}/~{slide.asap_slide_id}.NanoStringGeoMxSet.rds" if (dcc_to_rds_complete == "false") { call dcc_to_rds { input: team_id = team_id, dataset_id = dataset_id, - slide_id = slide.slide_id, + slide_id = slide.asap_slide_id, project_sample_metadata_csv = project_sample_metadata_csv, geomxngs_dcc_zip = geomxngs_dcc_zip_output, geomx_lab_annotation_xlsx = slide.geomx_lab_annotation_xlsx, @@ -121,15 +121,15 @@ workflow preprocess { File initial_rds_object_output = select_first([dcc_to_rds.initial_rds_object, dcc_to_rds_object]) #!FileCoercion - String qc_metrics_rds_object = "~{qc_raw_data_path}/~{slide.slide_id}.qc.rds" - String qc_segment_summary_csv = "~{qc_raw_data_path}/~{slide.slide_id}.segment_qc_summary.csv" - String qc_probe_summary_csv = "~{qc_raw_data_path}/~{slide.slide_id}.probe_qc_summary.csv" - String qc_gene_count_csv = "~{qc_raw_data_path}/~{slide.slide_id}.gene_count.csv" + String qc_metrics_rds_object = "~{qc_raw_data_path}/~{slide.asap_slide_id}.qc.rds" + String qc_segment_summary_csv = "~{qc_raw_data_path}/~{slide.asap_slide_id}.segment_qc_summary.csv" + String qc_probe_summary_csv = "~{qc_raw_data_path}/~{slide.asap_slide_id}.probe_qc_summary.csv" + String qc_gene_count_csv = "~{qc_raw_data_path}/~{slide.asap_slide_id}.gene_count.csv" if (qc_complete == "false") { call qc { input: - slide_id = slide.slide_id, + slide_id = slide.asap_slide_id, initial_rds_object = initial_rds_object_output, min_segment_reads = min_segment_reads, min_percent_reads_trimmed = min_percent_reads_trimmed, diff --git a/workflows/spatial_geomx/structs.wdl b/workflows/spatial_geomx/structs.wdl index e6a3635..08f44df 100644 --- a/workflows/spatial_geomx/structs.wdl +++ b/workflows/spatial_geomx/structs.wdl @@ -1,7 +1,7 @@ version 1.0 struct Sample { - String sample_id + String asap_sample_id String? batch Array[File]+ fastq_R1s @@ -11,19 +11,19 @@ struct Sample { } struct Slide { - String slide_id + String asap_slide_id File geomx_lab_annotation_xlsx Array[Sample] samples } struct Project { - String team_id - String dataset_id - String dataset_doi_url + String asap_team_id + String asap_dataset_id + String asap_dataset_doi_url Array[Slide] slides - File project_sample_metadata_csv + File asap_project_sample_metadata_csv File geomx_config_ini Boolean run_project_cohort_analysis From 5b26d5d9ad5e3cbc6dae83c3520d5d08ebc80991 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 10 Mar 2026 14:01:12 -0400 Subject: [PATCH 02/11] Save curated outputs in different path for GeoMx --- .../spatial_geomx/cohort_analysis/cohort_analysis.wdl | 8 +++++--- workflows/spatial_geomx/inputs.json | 2 +- workflows/spatial_geomx/main.wdl | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/workflows/spatial_geomx/cohort_analysis/cohort_analysis.wdl b/workflows/spatial_geomx/cohort_analysis/cohort_analysis.wdl index ff9086d..f5fb805 100644 --- a/workflows/spatial_geomx/cohort_analysis/cohort_analysis.wdl +++ b/workflows/spatial_geomx/cohort_analysis/cohort_analysis.wdl @@ -25,6 +25,7 @@ workflow cohort_analysis { String workflow_name String workflow_version String workflow_release + String crn_release_version String run_timestamp String raw_data_path_prefix Array[String] staging_data_buckets @@ -94,7 +95,7 @@ workflow cohort_analysis { input: output_file_paths = preprocessing_output_file_paths, staging_data_buckets = staging_data_buckets, - staging_data_path = "~{workflow_name}/preprocess", + staging_data_path = "~{workflow_name}/release/~{crn_release_version}/preprocess", billing_project = billing_project, zones = zones } @@ -103,7 +104,7 @@ workflow cohort_analysis { input: output_file_paths = processing_output_file_paths, staging_data_buckets = staging_data_buckets, - staging_data_path = "~{workflow_name}/process_to_adata", + staging_data_path = "~{workflow_name}/release/~{crn_release_version}/process_to_adata", billing_project = billing_project, zones = zones } @@ -132,7 +133,7 @@ workflow cohort_analysis { input: output_file_paths = cohort_analysis_final_output_paths, staging_data_buckets = staging_data_buckets, - staging_data_path = "~{workflow_name}/~{sub_workflow_name}", + staging_data_path = "~{workflow_name}/release/~{crn_release_version}/~{sub_workflow_name}", billing_project = billing_project, zones = zones } @@ -178,6 +179,7 @@ workflow cohort_analysis { workflow_name: {help: "Workflow name; stored in the file-level manifest and final manifest with all saved files."} workflow_version: {help: "Workflow version; stored in the file-level manifest and final manifest with all saved files."} workflow_release: {help: "GitHub release; stored in the file-level manifest and final manifest with all saved files."} + crn_release_version: {help: "CRN Cloud release version; used to organize outputs and for the CRN Cloud release."} run_timestamp: {help: "UTC timestamp; stored in the file-level manifest and final manifest with all saved files."} raw_data_path_prefix: {help: "Raw data bucket path prefix; location of raw bucket to upload task outputs to (`/workflow_execution/cohort_analysis`)."} staging_data_buckets: {help: "Array of staging data buckets to upload intermediate files to (i.e., DEV or UAT buckets depending on internal QC status)."} diff --git a/workflows/spatial_geomx/inputs.json b/workflows/spatial_geomx/inputs.json index 9eefe5d..0a8e5d6 100644 --- a/workflows/spatial_geomx/inputs.json +++ b/workflows/spatial_geomx/inputs.json @@ -1,5 +1,5 @@ { - "spatial_geomx_analysis.projects": "Array[WomCompositeType {\n slides -> Array[WomCompositeType {\n slide_id -> String\ngeomx_lab_annotation_xlsx -> File\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nsample_id -> String\nfastq_R1s -> Array[File]+\nbatch -> String?\nfastq_I2s -> Array[File]\nfastq_R2s -> Array[File]+ \n}] \n}]\nproject_sample_metadata_csv -> File\nteam_id -> String\ndataset_id -> String\ngeomx_config_ini -> File\nstaging_data_buckets -> Array[String]\nrun_project_cohort_analysis -> Boolean\ndataset_doi_url -> String\nraw_data_bucket -> String \n}]", + "spatial_geomx_analysis.projects": "Array[WomCompositeType {\n slides -> Array[WomCompositeType {\n asap_slide_id -> String\ngeomx_lab_annotation_xlsx -> File\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nfastq_R1s -> Array[File]+\nasap_sample_id -> String\nbatch -> String?\nfastq_I2s -> Array[File]\nfastq_R2s -> Array[File]+ \n}] \n}]\ngeomx_config_ini -> File\nrun_project_cohort_analysis -> Boolean\nraw_data_bucket -> String\nasap_team_id -> String\nasap_dataset_id -> String\nasap_project_sample_metadata_csv -> File\nstaging_data_buckets -> Array[String]\nasap_dataset_doi_url -> String \n}]", "spatial_geomx_analysis.geomxngs_config_pkc": "File", "spatial_geomx_analysis.min_segment_reads": "Int (optional, default = 1000)", "spatial_geomx_analysis.min_percent_reads_trimmed": "Int (optional, default = 80)", diff --git a/workflows/spatial_geomx/main.wdl b/workflows/spatial_geomx/main.wdl index cd88f46..111160d 100644 --- a/workflows/spatial_geomx/main.wdl +++ b/workflows/spatial_geomx/main.wdl @@ -43,6 +43,7 @@ workflow spatial_geomx_analysis { String workflow_name = "spatial_geomx" String workflow_version = "v1.0.0" String workflow_release = "https://github.com/ASAP-CRN/spatial-transcriptomics-wf/releases/tag/spatial_geomx_analysis-~{workflow_version}" + String crn_release_version = "v4.0.0" call GetWorkflowMetadata.get_workflow_metadata { input: @@ -127,6 +128,7 @@ workflow spatial_geomx_analysis { workflow_name = workflow_name, workflow_version = workflow_version, workflow_release = workflow_release, + crn_release_version = crn_release_version, run_timestamp = get_workflow_metadata.timestamp, raw_data_path_prefix = project_raw_data_path_prefix, staging_data_buckets = project.staging_data_buckets, From ee77e2595825605a7ac66d6ed50b7e030fc0a8c3 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 10 Mar 2026 14:05:15 -0400 Subject: [PATCH 03/11] Prefix IDs with asap in Visium workflow --- README.md | 20 ++++----- workflows/spatial_visium/main.wdl | 8 ++-- .../spatial_visium/preprocess/preprocess.wdl | 42 +++++++++---------- workflows/spatial_visium/structs.wdl | 8 ++-- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index a3853a4..32c2e2d 100644 --- a/README.md +++ b/README.md @@ -130,11 +130,11 @@ An input template file can be found at [workflows/spatial_visium/inputs.json](wo | Type | Name | Description | | :- | :- | :- | -| String | team_id | Unique identifier for team; used for naming output files. | -| String | dataset_id | Unique identifier for dataset; used for naming output files. | -| String | dataset_doi_url | Generated Zenodo DOI URL referencing the dataset. | +| String | asap_team_id | ASAP-generated unique identifier for team; used for naming output files. | +| String | asap_dataset_id | ASAP-generated unique identifier for dataset; used for naming output files. | +| String | asap_dataset_doi_url | ASAP-generated Zenodo DOI URL referencing the dataset. | | Array[[Slide](#nanostring-geomx-slide)] | slides | The set of slides associated with this project. | -| File | project_sample_metadata_csv | CSV containing all sample information including batch, condition, etc. | +| File | asap_project_sample_metadata_csv | ASAP-generated CSV containing all sample information including batch, condition, etc. | | File | geomx_config_ini | The configuration (.ini) file, containing pipeline processing parameters that is used by the GeoMx NGS pipeline to assist in converting the FASTQ files to DCC files. It is from the GeoMx DSP readout package. Sections can include `[Sequencing]`, `[Processing_v2]`, `[AOI_List]`, and `[Targets]`; see [GeoMx configuration (.ini) files notes](#geomx-configuration-(.ini)-files). | | Boolean | run_project_cohort_analysis | Whether or not to run cohort analysis within the project. | | String | raw_data_bucket | Raw data bucket; intermediate output files that are not final workflow outputs are stored here. | @@ -144,7 +144,7 @@ An input template file can be found at [workflows/spatial_visium/inputs.json](wo | Type | Name | Description | | :- | :- | :- | -| String | slide_id | Unique identifier for the slide within the project; used for naming output files. | +| String | asap_slide_id | ASAP-generated unique identifier for the slide within the project; used for naming output files. | | File | geomx_lab_annotation_xlsx | The annotation (.xlsx) file/lab worksheet, containing phenotypic data from the GeoMx DSP readout package; see [GeoMx Lab Worksheet notes](#geomx-lab-worksheet). | | Array[[Sample](#nanostring-geomx-sample)] | samples | The set of samples associated with this project. | @@ -152,7 +152,7 @@ An input template file can be found at [workflows/spatial_visium/inputs.json](wo | Type | Name | Description | | :- | :- | :- | -| String | sample_id | Unique identifier for the sample within the project. | +| String | asap_sample_id | ASAP-generated unique identifier for the sample within the project. | | String? | batch | The sample's batch. | | File | fastq_R1 | Path to the sample's read 1 FASTQ file. | | File | fastq_R2 | Path to the sample's read 2 FASTQ file. | @@ -165,9 +165,9 @@ An input template file can be found at [workflows/spatial_visium/inputs.json](wo | Type | Name | Description | | :- | :- | :- | -| String | team_id | Unique identifier for team; used for naming output files. | -| String | dataset_id | Unique identifier for dataset; used for naming output files. | -| String | dataset_doi_url | Generated Zenodo DOI URL referencing the dataset. | +| String | asap_team_id | ASAP-generated unique identifier for team; used for naming output files. | +| String | asap_dataset_id | ASAP-generated unique identifier for dataset; used for naming output files. | +| String | asap_dataset_doi_url | ASAP-generated Zenodo DOI URL referencing the dataset. | | Array[[Sample](#10x-visium-sample)] | samples | The set of samples associated with this project. | | Boolean | run_project_cohort_analysis | Whether or not to run cohort analysis within the project. | | String | raw_data_bucket | Raw data bucket; intermediate output files that are not final workflow outputs are stored here. | @@ -177,7 +177,7 @@ An input template file can be found at [workflows/spatial_visium/inputs.json](wo | Type | Name | Description | | :- | :- | :- | -| String | sample_id | Unique identifier for the sample within the project. | +| String | asap_sample_id | ASAP-generated unique identifier for the sample within the project. | | String? | batch | The sample's batch. | | File | fastq_R1 | Path to the sample's read 1 FASTQ file. | | File | fastq_R2 | Path to the sample's read 2 FASTQ file. | diff --git a/workflows/spatial_visium/main.wdl b/workflows/spatial_visium/main.wdl index 92cad4e..5a2910c 100644 --- a/workflows/spatial_visium/main.wdl +++ b/workflows/spatial_visium/main.wdl @@ -44,9 +44,9 @@ workflow spatial_visium_analysis { call Preprocess.preprocess { input: - team_id = project.team_id, - dataset_id = project.dataset_id, - dataset_doi_url = project.dataset_doi_url, + team_id = project.asap_team_id, + dataset_id = project.asap_dataset_id, + dataset_doi_url = project.asap_dataset_doi_url, samples = project.samples, spaceranger_reference_data = spaceranger_reference_data, visium_probe_set_csv = visium_probe_set_csv, @@ -77,7 +77,7 @@ workflow spatial_visium_analysis { if (project.run_project_cohort_analysis) { call CohortAnalysis.cohort_analysis as project_cohort_analysis { input: - cohort_id = project.team_id, + cohort_id = project.asap_team_id, project_sample_ids = preprocess.project_sample_ids, preprocessed_adata_objects = preprocess.qc_adata_object, preprocessing_output_file_paths = preprocessing_output_file_paths, diff --git a/workflows/spatial_visium/preprocess/preprocess.wdl b/workflows/spatial_visium/preprocess/preprocess.wdl index 3dbb3f2..9a93d03 100644 --- a/workflows/spatial_visium/preprocess/preprocess.wdl +++ b/workflows/spatial_visium/preprocess/preprocess.wdl @@ -38,9 +38,9 @@ workflow preprocess { String qc_raw_data_path = "~{workflow_raw_data_path_prefix}/qc/~{qc_task_version}" scatter (sample_object in samples) { - String spaceranger_count_output = "~{spaceranger_raw_data_path}/~{sample_object.sample_id}.raw_feature_bc_matrix.h5" - String counts_to_adata_output = "~{adata_raw_data_path}/~{sample_object.sample_id}.cleaned_unfiltered.h5ad" - String qc_output = "~{qc_raw_data_path}/~{sample_object.sample_id}.qc.h5ad" + String spaceranger_count_output = "~{spaceranger_raw_data_path}/~{sample_object.asap_sample_id}.raw_feature_bc_matrix.h5" + String counts_to_adata_output = "~{adata_raw_data_path}/~{sample_object.asap_sample_id}.cleaned_unfiltered.h5ad" + String qc_output = "~{qc_raw_data_path}/~{sample_object.asap_sample_id}.qc.h5ad" } # For each sample, outputs an array of true/false: [spaceranger_count_complete, counts_to_adata_complete, qc_complete] @@ -56,31 +56,31 @@ workflow preprocess { scatter (sample_index in range(length(samples))) { Sample sample = samples[sample_index] - Array[String] project_sample_id = [team_id, sample.sample_id, dataset_doi_url] + Array[String] project_sample_id = [team_id, sample.asap_sample_id, dataset_doi_url] String spaceranger_count_complete = check_output_files_exist.sample_preprocessing_complete[sample_index][0] String counts_to_adata_complete = check_output_files_exist.sample_preprocessing_complete[sample_index][1] String qc_complete = check_output_files_exist.sample_preprocessing_complete[sample_index][2] - String spaceranger_raw_counts = "~{spaceranger_raw_data_path}/~{sample.sample_id}.raw_feature_bc_matrix.h5" - String spaceranger_filtered_counts = "~{spaceranger_raw_data_path}/~{sample.sample_id}.filtered_feature_bc_matrix.h5" - String spaceranger_molecule_info = "~{spaceranger_raw_data_path}/~{sample.sample_id}.molecule_info.h5" - String spaceranger_metrics_summary_csv = "~{spaceranger_raw_data_path}/~{sample.sample_id}.metrics_summary.csv" - String spaceranger_spatial_outputs_tar_gz = "~{spaceranger_raw_data_path}/~{sample.sample_id}.spaceranger_spatial_outputs.tar.gz" + String spaceranger_raw_counts = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.raw_feature_bc_matrix.h5" + String spaceranger_filtered_counts = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.filtered_feature_bc_matrix.h5" + String spaceranger_molecule_info = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.molecule_info.h5" + String spaceranger_metrics_summary_csv = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.metrics_summary.csv" + String spaceranger_spatial_outputs_tar_gz = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.spaceranger_spatial_outputs.tar.gz" Array[String] spaceranger_spatial_images = [ - "~{spaceranger_raw_data_path}/~{sample.sample_id}.aligned_fiducials.jpg", - "~{spaceranger_raw_data_path}/~{sample.sample_id}.detected_tissue_image.jpg", - "~{spaceranger_raw_data_path}/~{sample.sample_id}.tissue_hires_image.png", - "~{spaceranger_raw_data_path}/~{sample.sample_id}.tissue_lowres_image.png" + "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.aligned_fiducials.jpg", + "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.detected_tissue_image.jpg", + "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.tissue_hires_image.png", + "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.tissue_lowres_image.png" ] - String spaceranger_scalefactors_json = "~{spaceranger_raw_data_path}/~{sample.sample_id}.scalefactors_json.json" - String spaceranger_tissue_positions_csv = "~{spaceranger_raw_data_path}/~{sample.sample_id}.tissue_positions.csv" - String spaceranger_spatial_enrichment_csv = "~{spaceranger_raw_data_path}/~{sample.sample_id}.spatial_enrichment.csv" + String spaceranger_scalefactors_json = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.scalefactors_json.json" + String spaceranger_tissue_positions_csv = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.tissue_positions.csv" + String spaceranger_spatial_enrichment_csv = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.spatial_enrichment.csv" if (spaceranger_count_complete == "false") { call spaceranger_count { input: - sample_id = sample.sample_id, + sample_id = sample.asap_sample_id, fastq_R1s = sample.fastq_R1s, fastq_R2s = sample.fastq_R2s, fastq_I1s = sample.fastq_I1s, @@ -108,14 +108,14 @@ workflow preprocess { File tissue_positions_csv_output = select_first([spaceranger_count.tissue_positions_csv, spaceranger_tissue_positions_csv]) #!FileCoercion File spatial_enrichment_csv_output = select_first([spaceranger_count.spatial_enrichment_csv, spaceranger_spatial_enrichment_csv]) #!FileCoercion - String counts_to_adata_object = "~{adata_raw_data_path}/~{sample.sample_id}.cleaned_unfiltered.h5ad" + String counts_to_adata_object = "~{adata_raw_data_path}/~{sample.asap_sample_id}.cleaned_unfiltered.h5ad" if (counts_to_adata_complete == "false") { call counts_to_adata { input: team_id = team_id, dataset_id = dataset_id, - sample_id = sample.sample_id, + sample_id = sample.asap_sample_id, batch = select_first([sample.batch]), visium_slide_serial_number = sample.visium_slide_serial_number, visium_capture_area = sample.visium_capture_area, @@ -130,12 +130,12 @@ workflow preprocess { File initial_adata_object_output = select_first([counts_to_adata.initial_adata_object, counts_to_adata_object]) #!FileCoercion - String qc_metrics_adata_object = "~{qc_raw_data_path}/~{sample.sample_id}.qc.h5ad" + String qc_metrics_adata_object = "~{qc_raw_data_path}/~{sample.asap_sample_id}.qc.h5ad" if (qc_complete == "false") { call qc { input: - sample_id = sample.sample_id, + sample_id = sample.asap_sample_id, initial_adata_object = initial_adata_object_output, raw_data_path = qc_raw_data_path, workflow_info = workflow_info, diff --git a/workflows/spatial_visium/structs.wdl b/workflows/spatial_visium/structs.wdl index 6bd8057..711caa5 100644 --- a/workflows/spatial_visium/structs.wdl +++ b/workflows/spatial_visium/structs.wdl @@ -1,7 +1,7 @@ version 1.0 struct Sample { - String sample_id + String asap_sample_id String? batch Array[File]+ fastq_R1s @@ -15,9 +15,9 @@ struct Sample { } struct Project { - String team_id - String dataset_id - String dataset_doi_url + String asap_team_id + String asap_dataset_id + String asap_dataset_doi_url Array[Sample] samples Boolean run_project_cohort_analysis From 60deab084d007de163cddabdc602ed572fb850d6 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 10 Mar 2026 14:08:40 -0400 Subject: [PATCH 04/11] Save curated outputs in different path for Visium --- .../spatial_visium/cohort_analysis/cohort_analysis.wdl | 6 ++++-- workflows/spatial_visium/inputs.json | 2 +- workflows/spatial_visium/main.wdl | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/workflows/spatial_visium/cohort_analysis/cohort_analysis.wdl b/workflows/spatial_visium/cohort_analysis/cohort_analysis.wdl index 6a3e69c..ed0e578 100644 --- a/workflows/spatial_visium/cohort_analysis/cohort_analysis.wdl +++ b/workflows/spatial_visium/cohort_analysis/cohort_analysis.wdl @@ -30,6 +30,7 @@ workflow cohort_analysis { String workflow_name String workflow_version String workflow_release + String crn_release_version String run_timestamp String raw_data_path_prefix Array[String] staging_data_buckets @@ -126,7 +127,7 @@ workflow cohort_analysis { input: output_file_paths = preprocessing_output_file_paths, staging_data_buckets = staging_data_buckets, - staging_data_path = "~{workflow_name}/preprocess", + staging_data_path = "~{workflow_name}/release/~{crn_release_version}/preprocess", billing_project = billing_project, zones = zones } @@ -163,7 +164,7 @@ workflow cohort_analysis { input: output_file_paths = cohort_analysis_final_output_paths, staging_data_buckets = staging_data_buckets, - staging_data_path = "~{workflow_name}/~{sub_workflow_name}", + staging_data_path = "~{workflow_name}/release/~{crn_release_version}/~{sub_workflow_name}", billing_project = billing_project, zones = zones } @@ -221,6 +222,7 @@ workflow cohort_analysis { workflow_name: {help: "Workflow name; stored in the file-level manifest and final manifest with all saved files."} workflow_version: {help: "Workflow version; stored in the file-level manifest and final manifest with all saved files."} workflow_release: {help: "GitHub release; stored in the file-level manifest and final manifest with all saved files."} + crn_release_version: {help: "CRN Cloud release version; used to organize outputs and for the CRN Cloud release."} run_timestamp: {help: "UTC timestamp; stored in the file-level manifest and final manifest with all saved files."} raw_data_path_prefix: {help: "Raw data bucket path prefix; location of raw bucket to upload task outputs to (`/workflow_execution/cohort_analysis`)."} staging_data_buckets: {help: "Array of staging data buckets to upload intermediate files to (i.e., DEV or UAT buckets depending on internal QC status)."} diff --git a/workflows/spatial_visium/inputs.json b/workflows/spatial_visium/inputs.json index 4bdf084..4546e08 100644 --- a/workflows/spatial_visium/inputs.json +++ b/workflows/spatial_visium/inputs.json @@ -1,5 +1,5 @@ { - "spatial_visium_analysis.projects": "Array[WomCompositeType {\n team_id -> String\ndataset_id -> String\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nsample_id -> String\nfastq_R1s -> Array[File]+\nbatch -> String?\nvisium_capture_area -> String\nvisium_brightfield_image -> File\nfastq_I2s -> Array[File]\nvisium_slide_serial_number -> String\nfastq_R2s -> Array[File]+ \n}]\nstaging_data_buckets -> Array[String]\nrun_project_cohort_analysis -> Boolean\ndataset_doi_url -> String\nraw_data_bucket -> String \n}]", + "spatial_visium_analysis.projects": "Array[WomCompositeType {\n samples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nfastq_R1s -> Array[File]+\nasap_sample_id -> String\nbatch -> String?\nvisium_capture_area -> String\nvisium_brightfield_image -> File\nfastq_I2s -> Array[File]\nvisium_slide_serial_number -> String\nfastq_R2s -> Array[File]+ \n}]\nasap_dataset_id -> String\nrun_project_cohort_analysis -> Boolean\nraw_data_bucket -> String\nasap_team_id -> String\nstaging_data_buckets -> Array[String]\nasap_dataset_doi_url -> String \n}]", "spatial_visium_analysis.spaceranger_reference_data": "File", "spatial_visium_analysis.visium_probe_set_csv": "File? (optional)", "spatial_visium_analysis.filter_cells_min_counts": "Int (optional, default = 5000)", diff --git a/workflows/spatial_visium/main.wdl b/workflows/spatial_visium/main.wdl index 5a2910c..56fc806 100644 --- a/workflows/spatial_visium/main.wdl +++ b/workflows/spatial_visium/main.wdl @@ -33,6 +33,7 @@ workflow spatial_visium_analysis { String workflow_name = "spatial_visium" String workflow_version = "v1.0.1" String workflow_release = "https://github.com/ASAP-CRN/spatial-transcriptomics-wf/releases/tag/spatial_visium_analysis-~{workflow_version}" + String crn_release_version = "v4.0.0" call GetWorkflowMetadata.get_workflow_metadata { input: @@ -93,6 +94,7 @@ workflow spatial_visium_analysis { workflow_name = workflow_name, workflow_version = workflow_version, workflow_release = workflow_release, + crn_release_version = crn_release_version, run_timestamp = get_workflow_metadata.timestamp, raw_data_path_prefix = project_raw_data_path_prefix, staging_data_buckets = project.staging_data_buckets, From ddacb5cae8949282884d152fbe6a5e20708a29c5 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 10 Mar 2026 15:45:37 -0400 Subject: [PATCH 05/11] Update README generate inputs --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 32c2e2d..365b870 100644 --- a/README.md +++ b/README.md @@ -189,9 +189,9 @@ An input template file can be found at [workflows/spatial_visium/inputs.json](wo ## Generating the inputs JSON -The inputs JSON may be generated manually, however when running a large number of samples, this can become unwieldly. The [`generate_inputs` utility script](https://github.com/ASAP-CRN/wf-common/blob/main/util/generate_inputs) may be used to automatically generate the inputs JSON (`inputs.{staging_env}.{source}-{cohort_dataset}.{date}.json`) and a sample list TSV (`{team_id}.{source}-{cohort_dataset}.sample_list.{date}.tsv`); same as the one generated in [the write_cohort_sample_list task](https://github.com/ASAP-CRN/wf-common/wdl/tasks/write_cohort_sample_list.wdl)). The script requires the libraries outlined in [the requirements.txt file](https://github.com/ASAP-CRN/wf-common/util/requirements.txt) and the following inputs: +The inputs JSON may be generated manually, however when running a large number of samples, this can become unwieldly. The [`generate_inputs` utility script](https://github.com/ASAP-CRN/wf-common/blob/main/util/generate_inputs) may be used to automatically generate the inputs JSON (`inputs.{staging_env}.{cohort_dataset_id}.{date}.json`) and a sample list TSV (`{team_id}.{cohort_dataset_id}.sample_list.{date}.tsv`); same as the one generated in [the write_cohort_sample_list task](https://github.com/ASAP-CRN/wf-common/wdl/tasks/write_cohort_sample_list.wdl)). The script requires the libraries outlined in [the requirements.txt file](https://github.com/ASAP-CRN/wf-common/util/requirements.txt) and the following inputs: -- `project-tsv`: One or more project TSVs with one row per sample and columns team_id, ASAP_dataset_id, ASAP_sample_id, batch, fastq_R1s, fastq_R2s, fastq_I1s, fastq_I2s, embargoed, source, dataset, dataset_DOI_url, and SPATIAL columns if applicable: geomx_config, geomx_dsp_config, geomx_annotation_file, visium_cytassist, visium_probe_set, visium_slide_ref, and visium_capture_area. All samples from all projects may be included in the same project TSV, or multiple project TSVs may be provided. +- `project-tsv`: One or more project TSVs with one row per sample and columns team_id, ASAP_dataset_id, ASAP_sample_id, batch, fastq_R1s, fastq_R2s, fastq_R3s, fastq_I1s, fastq_I2s, embargoed, source, modality_flavour, dataset_DOI_url, and SPATIAL columns if applicable: geomx_config, geomx_dsp_config, geomx_annotation_file, visium_cytassist, visium_probe_set, visium_slide_ref, and visium_capture_area. All samples from all projects may be included in the same project TSV, or multiple project TSVs may be provided. - `team_id`: A unique identifier for the team from which the sample(s) arose. - `ASAP_dataset_id`: A generated unique identifier for the dataset from which the sample(s) arose. - `ASAP_sample_id`: A generated unique identifier for the sample within the project. @@ -204,7 +204,7 @@ The inputs JSON may be generated manually, however when running a large number o - `fastq_I2s`: The gs uri to sample FASTQ index 2. - `embargoed`: The internal QC/embargo status of dataset. - `source`: The source of dataset (e.g. 'pmdbs'). - - `dataset`: The assigned dataset name without the source (e.g. 'sn-rnaseq') + - `modality_flavour`: The data modality flavour of dataset (e.g. 'sn-rnaseq') - `dataset_DOI_url`: Generated Zenodo DOI URL referencing the dataset. - `geomx_config`, `geomx_dsp_config`, `geomx_annotation_file`: go to [Nanostring GeoMx inputs](#nanostring-geomx-inputs) - `geomx_slide`: The GeoMx DSP identifier for the slide from which the sample(s) arose. @@ -213,7 +213,7 @@ The inputs JSON may be generated manually, however when running a large number o - `inputs-template`: The inputs template JSON file into which the `projects` information derived from the `project-tsv` will be inserted. Must have a key ending in `*.projects`. Other default values filled out in the inputs template will be written to the output inputs.json file. - `run-project-cohort-analysis`: Optionally run project-level cohort analysis for provided projects. This value will apply to all projects. [false] - `workflow_name`: WDL workflow name. -- `cohort-dataset`: Dataset name in cohort bucket name (e.g. 'sc-rnaseq'). +- `cohort-dataset-id`: Dataset name in cohort bucket id (e.g. 'cohort-pmdbs-sc-rnaseq'). Example usage: @@ -222,15 +222,13 @@ Example usage: --project-tsv metadata.tsv \ --inputs-template workflows/spatial_geomx/inputs.json \ --run-project-cohort-analysis \ - --workflow-name spatial_geomx_analysis \ - --cohort-dataset spatial-geomx + --workflow-name spatial_geomx_analysis ./wf-common/util/generate_inputs \ --project-tsv metadata.tsv \ --inputs-template workflows/spatial_visium/inputs.json \ --run-project-cohort-analysis \ - --workflow-name spatial_visium_analysis \ - --cohort-dataset spatial-visium + --workflow-name spatial_visium_analysis ``` # Outputs From e5c82db7b421e4ad644c21e71a828807128efe93 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 10 Mar 2026 15:55:43 -0400 Subject: [PATCH 06/11] Update README --- README.md | 208 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 109 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 365b870..df17286 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,7 @@ The raw data bucket will contain *some* artifacts generated as part of workflow In the workflow, task outputs are either specified as `String` (final outputs, which will be copied in order to live in raw data buckets and staging buckets) or `File` (intermediate outputs that are periodically cleaned up, which will live in the cromwell-output bucket). This was implemented to reduce storage costs. ```bash -asap-raw-{cohort,team-xxyy}-{source}-{dataset} +asap-raw-{cohort,team-xxyy}-{source}-{modality_flavour}-{context} └── workflow_execution └── spatial_geomx ├── cohort_analysis @@ -269,7 +269,7 @@ asap-raw-{cohort,team-xxyy}-{source}-{dataset} └── ${qc_task_version} └── -asap-raw-{cohort,team-xxyy}-{source}-{dataset} +asap-raw-{cohort,team-xxyy}-{source}-{modality_flavour}-{context} └── workflow_execution └── spatial_visium ├── cohort_analysis @@ -290,99 +290,111 @@ asap-raw-{cohort,team-xxyy}-{source}-{dataset} ### Staging data (intermediate workflow objects and final workflow outputs for the latest run of the workflow) -Following QC by researchers, the objects in the dev or uat bucket are synced into the curated data buckets, maintaining the same file structure. Curated data buckets are named `asap-curated-{team-xxyy}-{source}-{dataset}`. +Following QC by researchers, the objects in the dev or uat bucket are synced into the curated data buckets, maintaining the same file structure. Curated data buckets are named `asap-curated-{cohort,team-xxyy}-{source}-{modality_flavour}-{context}` and `dataset_id` = `{cohort,team-xxyy}-{source}-{modality_flavour}-{context}`. Data may be synced using [the `promote_staging_data` script](#promoting-staging-data). ```bash -asap-dev-{team-xxyy}-{source}-{dataset} +asap-dev-{team-xxyy}-{source}-{modality_flavour}-{context} └── spatial_geomx - ├── cohort_analysis - │ ├── ${team_id}.sample_list.tsv - │ ├── ${team_id}.merged_metadata.csv - │ ├── ${team_id}.merged_processed.h5ad - │ ├── ${team_id}.all_genes.csv - │ ├── ${team_id}.hvg_genes.csv - │ ├── ${team_id}.hvg_dispersion.png - │ ├── ${team_id}.umap_cluster.png - │ ├── ${team_id}.final.h5ad - │ ├── ${team_id}.final_metadata.csv - │ └── MANIFEST.tsv - ├── process_to_adata - │ ├── ${slideN_id}.segment_gene_detection_plot.png - │ ├── ${slideN_id}.gene_detection_rate.csv - │ ├── ${slideN_id}.q3_negprobe_plot.png - │ ├── ${slideN_id}.normalization_plot.png - │ └── MANIFEST.tsv - └── preprocess - ├── ${slideA_id}.DCC.zip - ├── ${slideA_id}.geomxngs_out_dir.tar.gz - ├── ${slideA_id}.NanoStringGeoMxSet.rds - ├── ${slideA_id}.qc.rds - ├── ${slideA_id}.segment_qc_summary.csv - ├── ${slideA_id}.probe_qc_summary.csv - ├── ${slideA_id}.gene_count.csv - ├── MANIFEST.tsv - ├── ... - ├── ${slideN_id}.DCC.zip - ├── ${slideN_id}.geomxngs_out_dir.tar.gz - ├── ${slideN_id}.NanoStringGeoMxSet.rds - ├── ${slideN_id}.qc.rds - ├── ${slideN_id}.segment_qc_summary.csv - ├── ${slideN_id}.probe_qc_summary.csv - ├── ${slideN_id}.gene_count.csv - └── MANIFEST.tsv - -asap-dev-{team-xxyy}-{source}-{dataset} + └── release + └── ${crn_release_version} + ├── cohort_analysis + │ ├── ${team_id}.sample_list.tsv + │ ├── ${team_id}.merged_metadata.csv + │ ├── ${team_id}.merged_processed.h5ad + │ ├── ${team_id}.all_genes.csv + │ ├── ${team_id}.hvg_genes.csv + │ ├── ${team_id}.hvg_dispersion.png + │ ├── ${team_id}.umap_cluster.png + │ ├── ${team_id}.final.h5ad + │ ├── ${team_id}.final_metadata.csv + │ └── MANIFEST.tsv + ├── process_to_adata + │ ├── ${slideN_id}.segment_gene_detection_plot.png + │ ├── ${slideN_id}.gene_detection_rate.csv + │ ├── ${slideN_id}.q3_negprobe_plot.png + │ ├── ${slideN_id}.normalization_plot.png + │ └── MANIFEST.tsv + ├── preprocess + │ ├── ${slideA_id}.DCC.zip + │ ├── ${slideA_id}.geomxngs_out_dir.tar.gz + │ ├── ${slideA_id}.NanoStringGeoMxSet.rds + │ ├── ${slideA_id}.qc.rds + │ ├── ${slideA_id}.segment_qc_summary.csv + │ ├── ${slideA_id}.probe_qc_summary.csv + │ ├── ${slideA_id}.gene_count.csv + │ ├── ... + │ ├── ${slideN_id}.DCC.zip + │ ├── ${slideN_id}.geomxngs_out_dir.tar.gz + │ ├── ${slideN_id}.NanoStringGeoMxSet.rds + │ ├── ${slideN_id}.qc.rds + │ ├── ${slideN_id}.segment_qc_summary.csv + │ ├── ${slideN_id}.probe_qc_summary.csv + │ ├── ${slideN_id}.gene_count.csv + │ └── MANIFEST.tsv + ├── workflow_version # plain text file + └── workflow_metadata + └── ${timestamp} + ├── MANIFEST.tsv # combined + └── data_promotion_report.md + +asap-dev-{team-xxyy}-{source}-{modality_flavour}-{context} └── spatial_visium - ├── cohort_analysis - │ ├── ${team_id}.sample_list.tsv - │ ├── ${team_id}.merged_cleaned_unfiltered.h5ad - │ ├── ${team_id}.merged_metadata.csv - │ ├── ${team_id}.all_genes.csv - │ ├── ${team_id}.hvg_genes.csv - │ ├── ${team_id}.qc_violin.png - │ ├── ${team_id}.qc_dist.png - │ ├── ${team_id}.hvg_dispersion.png - │ ├── ${team_id}.umap_cluster.png - │ ├── ${team_id}.spatial_scatter.png - │ ├── ${team_id}.final.h5ad - │ ├── ${team_id}.final_metadata.csv - │ ├── ${team_id}.moran_top_10_variable_genes.csv - │ ├── ${team_id}.moran_top_4_variable_genes_spatial_scatter.png - │ └── MANIFEST.tsv - └── preprocess - ├── ${sampleA_id}.raw_feature_bc_matrix.h5 - ├── ${sampleA_id}.filtered_feature_bc_matrix.h5 - ├── ${sampleA_id}.molecule_info.h5 - ├── ${sampleA_id}.metrics_summary.csv - ├── ${sampleA_id}.spaceranger_spatial_outputs.tar.gz - ├── ${sampleA_id}.aligned_fiducials.jpg - ├── ${sampleA_id}.detected_tissue_image.jpg - ├── ${sampleA_id}.tissue_hires_image.png - ├── ${sampleA_id}.tissue_lowres_image.png - ├── ${sampleA_id}.scalefactors_json.json - ├── ${sampleA_id}.tissue_positions.csv - ├── ${sampleA_id}.spatial_enrichment.csv - ├── ${sampleA_id}.cleaned_unfiltered.h5ad - ├── ${sampleA_id}.qc.h5ad - ├── MANIFEST.tsv - ├── ... - ├── ${sampleN_id}.raw_feature_bc_matrix.h5 - ├── ${sampleN_id}.filtered_feature_bc_matrix.h5 - ├── ${sampleN_id}.molecule_info.h5 - ├── ${sampleN_id}.metrics_summary.csv - ├── ${sampleN_id}.spaceranger_spatial_outputs.tar.gz - ├── ${sampleN_id}.aligned_fiducials.jpg - ├── ${sampleN_id}.detected_tissue_image.jpg - ├── ${sampleN_id}.tissue_hires_image.png - ├── ${sampleN_id}.tissue_lowres_image.png - ├── ${sampleN_id}.scalefactors_json.json - ├── ${sampleN_id}.tissue_positions.csv - ├── ${sampleN_id}.spatial_enrichment.csv - ├── ${sampleN_id}.cleaned_unfiltered.h5ad - ├── ${sampleN_id}.qc.h5ad - └── MANIFEST.tsv + └── release + └── ${crn_release_version} + ├── cohort_analysis + │ ├── ${team_id}.sample_list.tsv + │ ├── ${team_id}.merged_cleaned_unfiltered.h5ad + │ ├── ${team_id}.merged_metadata.csv + │ ├── ${team_id}.all_genes.csv + │ ├── ${team_id}.hvg_genes.csv + │ ├── ${team_id}.qc_violin.png + │ ├── ${team_id}.qc_dist.png + │ ├── ${team_id}.hvg_dispersion.png + │ ├── ${team_id}.umap_cluster.png + │ ├── ${team_id}.spatial_scatter.png + │ ├── ${team_id}.final.h5ad + │ ├── ${team_id}.final_metadata.csv + │ ├── ${team_id}.moran_top_10_variable_genes.csv + │ ├── ${team_id}.moran_top_4_variable_genes_spatial_scatter.png + │ └── MANIFEST.tsv + ├── preprocess + │ ├── ${sampleA_id}.raw_feature_bc_matrix.h5 + │ ├── ${sampleA_id}.filtered_feature_bc_matrix.h5 + │ ├── ${sampleA_id}.molecule_info.h5 + │ ├── ${sampleA_id}.metrics_summary.csv + │ ├── ${sampleA_id}.spaceranger_spatial_outputs.tar.gz + │ ├── ${sampleA_id}.aligned_fiducials.jpg + │ ├── ${sampleA_id}.detected_tissue_image.jpg + │ ├── ${sampleA_id}.tissue_hires_image.png + │ ├── ${sampleA_id}.tissue_lowres_image.png + │ ├── ${sampleA_id}.scalefactors_json.json + │ ├── ${sampleA_id}.tissue_positions.csv + │ ├── ${sampleA_id}.spatial_enrichment.csv + │ ├── ${sampleA_id}.cleaned_unfiltered.h5ad + │ ├── ${sampleA_id}.qc.h5ad + │ ├── ... + │ ├── ${sampleN_id}.raw_feature_bc_matrix.h5 + │ ├── ${sampleN_id}.filtered_feature_bc_matrix.h5 + │ ├── ${sampleN_id}.molecule_info.h5 + │ ├── ${sampleN_id}.metrics_summary.csv + │ ├── ${sampleN_id}.spaceranger_spatial_outputs.tar.gz + │ ├── ${sampleN_id}.aligned_fiducials.jpg + │ ├── ${sampleN_id}.detected_tissue_image.jpg + │ ├── ${sampleN_id}.tissue_hires_image.png + │ ├── ${sampleN_id}.tissue_lowres_image.png + │ ├── ${sampleN_id}.scalefactors_json.json + │ ├── ${sampleN_id}.tissue_positions.csv + │ ├── ${sampleN_id}.spatial_enrichment.csv + │ ├── ${sampleN_id}.cleaned_unfiltered.h5ad + │ ├── ${sampleN_id}.qc.h5ad + │ ├── MANIFEST.tsv + ├── workflow_version # plain text file + └── workflow_metadata + └── ${timestamp} + ├── MANIFEST.tsv # combined + └── data_promotion_report.md ``` ## Promoting staging data @@ -401,11 +413,9 @@ The script defaults to a dry run, printing out the files that would be copied or ``` -h Display this message and exit --t Space-delimited team(s) to promote data for -l List available teams --s Source name in bucket name --d Space-delimited dataset name(s) in team bucket name, must follow the same order as {team} --w Workflow name used as a directory in bucket +-w Workflow name used as a directory in bucket (e.g. 'spatial_visium') +-v Release version (e.g. v4.0.0) -p Promote data. If this option is not selected, data that would be copied or deleted is printed out, but files are not actually changed (dry run) ``` @@ -413,14 +423,14 @@ The script defaults to a dry run, printing out the files that would be copied or ```bash # List available teams -./wf-common/util/promote_staging_data -t cohort -l -s pmdbs -d spatial-geomx -w spatial_geomx -./wf-common/util/promote_staging_data -t cohort -l -s mouse -d spatial-visium -w spatial_visium +./wf-common/util/promote_staging_data -l -w spatial_geomx -v v4.0.0 +./wf-common/util/promote_staging_data -l -w spatial_visium -v v4.0.0 -# Print out the files that would be copied or deleted from the staging bucket to the curated bucket for teams team-edwards and team-vila -./wf-common/util/promote_staging_data -t team-edwards team-vila -s pmdbs -d spatial-geomx-th spatial-geomx-thlc -w spatial_geomx +# Print out the files that would be copied or deleted from the staging bucket to the curated bucket for teams' datasets processed through the spatial visium pipeline for a specific release version +./wf-common/util/promote_staging_data -w spatial_visium -v v4.0.0 -# Promote data for team-edwards and team-vila -./wf-common/util/promote_staging_data -t team-edwards team-vila -s pmdbs -d spatial-geomx-th spatial-geomx-thlc -w spatial_geomx -p +# Promote data for teams' datasets processed through the spatial visium pipeline for a specific release version +./wf-common/util/promote_staging_data -w spatial_visium -v v4.0.0 -p ``` # Docker images From 7f61ba4d87ff61b0841e498725c35538b1998e71 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Wed, 11 Mar 2026 14:39:19 -0400 Subject: [PATCH 07/11] Fix typo --- workflows/spatial_geomx/main.wdl | 2 +- workflows/spatial_visium/main.wdl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/spatial_geomx/main.wdl b/workflows/spatial_geomx/main.wdl index 111160d..30a1466 100644 --- a/workflows/spatial_geomx/main.wdl +++ b/workflows/spatial_geomx/main.wdl @@ -184,7 +184,7 @@ workflow spatial_geomx_analysis { } meta { - description: "Harmonized human postmortem-derived brain sequencing (PMDBS) and non-human spatial transcriptomics workflow for Nanostring GeoMx data" + description: "Harmonized human postmortem-derived brain sequencing (PMDBS) and non-human spatial transcriptomics workflow for Nanostring GeoMx data." } parameter_meta { diff --git a/workflows/spatial_visium/main.wdl b/workflows/spatial_visium/main.wdl index 56fc806..52ec48b 100644 --- a/workflows/spatial_visium/main.wdl +++ b/workflows/spatial_visium/main.wdl @@ -153,7 +153,7 @@ workflow spatial_visium_analysis { } meta { - description: "Harmonized human postmortem-derived brain sequencing (PMDBS) and non-human spatial transcriptomics workflow for 10x Visium data" + description: "Harmonized human postmortem-derived brain sequencing (PMDBS) and non-human spatial transcriptomics workflow for 10x Visium data." } parameter_meta { From c00e06a1c4232b30049f7f1f2e53effb97dd733c Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Fri, 13 Mar 2026 13:37:01 -0400 Subject: [PATCH 08/11] Update generate_inputs to include release_version arg in README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index df17286..5ff0a3f 100644 --- a/README.md +++ b/README.md @@ -222,12 +222,14 @@ Example usage: --project-tsv metadata.tsv \ --inputs-template workflows/spatial_geomx/inputs.json \ --run-project-cohort-analysis \ + --release-version v5.0.0 \ --workflow-name spatial_geomx_analysis ./wf-common/util/generate_inputs \ --project-tsv metadata.tsv \ --inputs-template workflows/spatial_visium/inputs.json \ --run-project-cohort-analysis \ + --release-version v5.0.0 \ --workflow-name spatial_visium_analysis ``` From 8f047b67db9a3a84f1fcaab576c1b10563d6cec3 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Thu, 19 Mar 2026 12:54:03 -0400 Subject: [PATCH 09/11] Revert to sample_id since replicate is appended --- README.md | 4 +- wf-common | 2 +- workflows/spatial_geomx/inputs.json | 2 +- .../spatial_geomx/preprocess/preprocess.wdl | 2 +- workflows/spatial_geomx/structs.wdl | 2 +- workflows/spatial_visium/inputs.json | 2 +- .../spatial_visium/preprocess/preprocess.wdl | 42 +++++++++---------- workflows/spatial_visium/structs.wdl | 2 +- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 5ff0a3f..687a4d1 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ An input template file can be found at [workflows/spatial_visium/inputs.json](wo | Type | Name | Description | | :- | :- | :- | -| String | asap_sample_id | ASAP-generated unique identifier for the sample within the project. | +| String | sample_id | ASAP-generated unique identifier combined with the replicate for the sample within the project. | | String? | batch | The sample's batch. | | File | fastq_R1 | Path to the sample's read 1 FASTQ file. | | File | fastq_R2 | Path to the sample's read 2 FASTQ file. | @@ -177,7 +177,7 @@ An input template file can be found at [workflows/spatial_visium/inputs.json](wo | Type | Name | Description | | :- | :- | :- | -| String | asap_sample_id | ASAP-generated unique identifier for the sample within the project. | +| String | sample_id | ASAP-generated unique identifier combined with the replicate for the sample within the project. | | String? | batch | The sample's batch. | | File | fastq_R1 | Path to the sample's read 1 FASTQ file. | | File | fastq_R2 | Path to the sample's read 2 FASTQ file. | diff --git a/wf-common b/wf-common index 1544966..39535eb 160000 --- a/wf-common +++ b/wf-common @@ -1 +1 @@ -Subproject commit 1544966acd53b0de5b0a71656e536d08a6496811 +Subproject commit 39535ebee6a1a428d07121292032a19f6bf52d2d diff --git a/workflows/spatial_geomx/inputs.json b/workflows/spatial_geomx/inputs.json index 0a8e5d6..0d2d0e3 100644 --- a/workflows/spatial_geomx/inputs.json +++ b/workflows/spatial_geomx/inputs.json @@ -1,5 +1,5 @@ { - "spatial_geomx_analysis.projects": "Array[WomCompositeType {\n slides -> Array[WomCompositeType {\n asap_slide_id -> String\ngeomx_lab_annotation_xlsx -> File\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nfastq_R1s -> Array[File]+\nasap_sample_id -> String\nbatch -> String?\nfastq_I2s -> Array[File]\nfastq_R2s -> Array[File]+ \n}] \n}]\ngeomx_config_ini -> File\nrun_project_cohort_analysis -> Boolean\nraw_data_bucket -> String\nasap_team_id -> String\nasap_dataset_id -> String\nasap_project_sample_metadata_csv -> File\nstaging_data_buckets -> Array[String]\nasap_dataset_doi_url -> String \n}]", + "spatial_geomx_analysis.projects": "Array[WomCompositeType {\n slides -> Array[WomCompositeType {\n asap_slide_id -> String\ngeomx_lab_annotation_xlsx -> File\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nfastq_R1s -> Array[File]+\nsample_id -> String\nbatch -> String?\nfastq_I2s -> Array[File]\nfastq_R2s -> Array[File]+ \n}] \n}]\ngeomx_config_ini -> File\nrun_project_cohort_analysis -> Boolean\nraw_data_bucket -> String\nasap_team_id -> String\nasap_dataset_id -> String\nasap_project_sample_metadata_csv -> File\nstaging_data_buckets -> Array[String]\nasap_dataset_doi_url -> String \n}]", "spatial_geomx_analysis.geomxngs_config_pkc": "File", "spatial_geomx_analysis.min_segment_reads": "Int (optional, default = 1000)", "spatial_geomx_analysis.min_percent_reads_trimmed": "Int (optional, default = 80)", diff --git a/workflows/spatial_geomx/preprocess/preprocess.wdl b/workflows/spatial_geomx/preprocess/preprocess.wdl index d3050c2..0e5cd7b 100644 --- a/workflows/spatial_geomx/preprocess/preprocess.wdl +++ b/workflows/spatial_geomx/preprocess/preprocess.wdl @@ -69,7 +69,7 @@ workflow preprocess { scatter (sample_index in range(length(slide.samples))) { Sample sample = slide.samples[sample_index] - Array[String] project_sample_id = [team_id, sample.asap_sample_id, dataset_doi_url] + Array[String] project_sample_id = [team_id, sample.sample_id, dataset_doi_url] Array[File] fastq_R1s = sample.fastq_R1s Array[File] fastq_R2s = sample.fastq_R2s } diff --git a/workflows/spatial_geomx/structs.wdl b/workflows/spatial_geomx/structs.wdl index 08f44df..4a95824 100644 --- a/workflows/spatial_geomx/structs.wdl +++ b/workflows/spatial_geomx/structs.wdl @@ -1,7 +1,7 @@ version 1.0 struct Sample { - String asap_sample_id + String sample_id String? batch Array[File]+ fastq_R1s diff --git a/workflows/spatial_visium/inputs.json b/workflows/spatial_visium/inputs.json index 4546e08..765bc63 100644 --- a/workflows/spatial_visium/inputs.json +++ b/workflows/spatial_visium/inputs.json @@ -1,5 +1,5 @@ { - "spatial_visium_analysis.projects": "Array[WomCompositeType {\n samples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nfastq_R1s -> Array[File]+\nasap_sample_id -> String\nbatch -> String?\nvisium_capture_area -> String\nvisium_brightfield_image -> File\nfastq_I2s -> Array[File]\nvisium_slide_serial_number -> String\nfastq_R2s -> Array[File]+ \n}]\nasap_dataset_id -> String\nrun_project_cohort_analysis -> Boolean\nraw_data_bucket -> String\nasap_team_id -> String\nstaging_data_buckets -> Array[String]\nasap_dataset_doi_url -> String \n}]", + "spatial_visium_analysis.projects": "Array[WomCompositeType {\n samples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nfastq_R1s -> Array[File]+\nsample_id -> String\nbatch -> String?\nvisium_capture_area -> String\nvisium_brightfield_image -> File\nfastq_I2s -> Array[File]\nvisium_slide_serial_number -> String\nfastq_R2s -> Array[File]+ \n}]\nasap_dataset_id -> String\nrun_project_cohort_analysis -> Boolean\nraw_data_bucket -> String\nasap_team_id -> String\nstaging_data_buckets -> Array[String]\nasap_dataset_doi_url -> String \n}]", "spatial_visium_analysis.spaceranger_reference_data": "File", "spatial_visium_analysis.visium_probe_set_csv": "File? (optional)", "spatial_visium_analysis.filter_cells_min_counts": "Int (optional, default = 5000)", diff --git a/workflows/spatial_visium/preprocess/preprocess.wdl b/workflows/spatial_visium/preprocess/preprocess.wdl index 9a93d03..3dbb3f2 100644 --- a/workflows/spatial_visium/preprocess/preprocess.wdl +++ b/workflows/spatial_visium/preprocess/preprocess.wdl @@ -38,9 +38,9 @@ workflow preprocess { String qc_raw_data_path = "~{workflow_raw_data_path_prefix}/qc/~{qc_task_version}" scatter (sample_object in samples) { - String spaceranger_count_output = "~{spaceranger_raw_data_path}/~{sample_object.asap_sample_id}.raw_feature_bc_matrix.h5" - String counts_to_adata_output = "~{adata_raw_data_path}/~{sample_object.asap_sample_id}.cleaned_unfiltered.h5ad" - String qc_output = "~{qc_raw_data_path}/~{sample_object.asap_sample_id}.qc.h5ad" + String spaceranger_count_output = "~{spaceranger_raw_data_path}/~{sample_object.sample_id}.raw_feature_bc_matrix.h5" + String counts_to_adata_output = "~{adata_raw_data_path}/~{sample_object.sample_id}.cleaned_unfiltered.h5ad" + String qc_output = "~{qc_raw_data_path}/~{sample_object.sample_id}.qc.h5ad" } # For each sample, outputs an array of true/false: [spaceranger_count_complete, counts_to_adata_complete, qc_complete] @@ -56,31 +56,31 @@ workflow preprocess { scatter (sample_index in range(length(samples))) { Sample sample = samples[sample_index] - Array[String] project_sample_id = [team_id, sample.asap_sample_id, dataset_doi_url] + Array[String] project_sample_id = [team_id, sample.sample_id, dataset_doi_url] String spaceranger_count_complete = check_output_files_exist.sample_preprocessing_complete[sample_index][0] String counts_to_adata_complete = check_output_files_exist.sample_preprocessing_complete[sample_index][1] String qc_complete = check_output_files_exist.sample_preprocessing_complete[sample_index][2] - String spaceranger_raw_counts = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.raw_feature_bc_matrix.h5" - String spaceranger_filtered_counts = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.filtered_feature_bc_matrix.h5" - String spaceranger_molecule_info = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.molecule_info.h5" - String spaceranger_metrics_summary_csv = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.metrics_summary.csv" - String spaceranger_spatial_outputs_tar_gz = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.spaceranger_spatial_outputs.tar.gz" + String spaceranger_raw_counts = "~{spaceranger_raw_data_path}/~{sample.sample_id}.raw_feature_bc_matrix.h5" + String spaceranger_filtered_counts = "~{spaceranger_raw_data_path}/~{sample.sample_id}.filtered_feature_bc_matrix.h5" + String spaceranger_molecule_info = "~{spaceranger_raw_data_path}/~{sample.sample_id}.molecule_info.h5" + String spaceranger_metrics_summary_csv = "~{spaceranger_raw_data_path}/~{sample.sample_id}.metrics_summary.csv" + String spaceranger_spatial_outputs_tar_gz = "~{spaceranger_raw_data_path}/~{sample.sample_id}.spaceranger_spatial_outputs.tar.gz" Array[String] spaceranger_spatial_images = [ - "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.aligned_fiducials.jpg", - "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.detected_tissue_image.jpg", - "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.tissue_hires_image.png", - "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.tissue_lowres_image.png" + "~{spaceranger_raw_data_path}/~{sample.sample_id}.aligned_fiducials.jpg", + "~{spaceranger_raw_data_path}/~{sample.sample_id}.detected_tissue_image.jpg", + "~{spaceranger_raw_data_path}/~{sample.sample_id}.tissue_hires_image.png", + "~{spaceranger_raw_data_path}/~{sample.sample_id}.tissue_lowres_image.png" ] - String spaceranger_scalefactors_json = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.scalefactors_json.json" - String spaceranger_tissue_positions_csv = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.tissue_positions.csv" - String spaceranger_spatial_enrichment_csv = "~{spaceranger_raw_data_path}/~{sample.asap_sample_id}.spatial_enrichment.csv" + String spaceranger_scalefactors_json = "~{spaceranger_raw_data_path}/~{sample.sample_id}.scalefactors_json.json" + String spaceranger_tissue_positions_csv = "~{spaceranger_raw_data_path}/~{sample.sample_id}.tissue_positions.csv" + String spaceranger_spatial_enrichment_csv = "~{spaceranger_raw_data_path}/~{sample.sample_id}.spatial_enrichment.csv" if (spaceranger_count_complete == "false") { call spaceranger_count { input: - sample_id = sample.asap_sample_id, + sample_id = sample.sample_id, fastq_R1s = sample.fastq_R1s, fastq_R2s = sample.fastq_R2s, fastq_I1s = sample.fastq_I1s, @@ -108,14 +108,14 @@ workflow preprocess { File tissue_positions_csv_output = select_first([spaceranger_count.tissue_positions_csv, spaceranger_tissue_positions_csv]) #!FileCoercion File spatial_enrichment_csv_output = select_first([spaceranger_count.spatial_enrichment_csv, spaceranger_spatial_enrichment_csv]) #!FileCoercion - String counts_to_adata_object = "~{adata_raw_data_path}/~{sample.asap_sample_id}.cleaned_unfiltered.h5ad" + String counts_to_adata_object = "~{adata_raw_data_path}/~{sample.sample_id}.cleaned_unfiltered.h5ad" if (counts_to_adata_complete == "false") { call counts_to_adata { input: team_id = team_id, dataset_id = dataset_id, - sample_id = sample.asap_sample_id, + sample_id = sample.sample_id, batch = select_first([sample.batch]), visium_slide_serial_number = sample.visium_slide_serial_number, visium_capture_area = sample.visium_capture_area, @@ -130,12 +130,12 @@ workflow preprocess { File initial_adata_object_output = select_first([counts_to_adata.initial_adata_object, counts_to_adata_object]) #!FileCoercion - String qc_metrics_adata_object = "~{qc_raw_data_path}/~{sample.asap_sample_id}.qc.h5ad" + String qc_metrics_adata_object = "~{qc_raw_data_path}/~{sample.sample_id}.qc.h5ad" if (qc_complete == "false") { call qc { input: - sample_id = sample.asap_sample_id, + sample_id = sample.sample_id, initial_adata_object = initial_adata_object_output, raw_data_path = qc_raw_data_path, workflow_info = workflow_info, diff --git a/workflows/spatial_visium/structs.wdl b/workflows/spatial_visium/structs.wdl index 711caa5..140ce1c 100644 --- a/workflows/spatial_visium/structs.wdl +++ b/workflows/spatial_visium/structs.wdl @@ -1,7 +1,7 @@ version 1.0 struct Sample { - String asap_sample_id + String sample_id String? batch Array[File]+ fastq_R1s From 4fef07ed58d6f8fa7983d9b8e38b443f3163de7e Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Wed, 22 Apr 2026 11:57:35 -0400 Subject: [PATCH 10/11] Update wf-common --- wf-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wf-common b/wf-common index 39535eb..e5e6e64 160000 --- a/wf-common +++ b/wf-common @@ -1 +1 @@ -Subproject commit 39535ebee6a1a428d07121292032a19f6bf52d2d +Subproject commit e5e6e64abaf2b67b71138dffc8c4851684053b86 From 5012fdd99f03ebd7b9de0cc1588eb48315f317ca Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Fri, 24 Apr 2026 09:30:35 -0400 Subject: [PATCH 11/11] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 687a4d1..4f38ada 100644 --- a/README.md +++ b/README.md @@ -391,7 +391,7 @@ asap-dev-{team-xxyy}-{source}-{modality_flavour}-{context} │ ├── ${sampleN_id}.spatial_enrichment.csv │ ├── ${sampleN_id}.cleaned_unfiltered.h5ad │ ├── ${sampleN_id}.qc.h5ad - │ ├── MANIFEST.tsv + │ └── MANIFEST.tsv ├── workflow_version # plain text file └── workflow_metadata └── ${timestamp}