Skip to content

Feature/improve id encoding and curated path #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kfang4 merged 11 commits into from
Apr 24, 2026
Merged

Feature/improve id encoding and curated path #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
2973586
Prefix IDs with asap in GeoMx workflow
kfang4 Mar 10, 2026
5b26d5d
Save curated outputs in different path for GeoMx
kfang4 Mar 10, 2026
ee77e25
Prefix IDs with asap in Visium workflow
kfang4 Mar 10, 2026
60deab0
Save curated outputs in different path for Visium
kfang4 Mar 10, 2026
ddacb5c
Update README generate inputs
kfang4 Mar 10, 2026
e5c82db
Update README
kfang4 Mar 10, 2026
7f61ba4
Fix typo
kfang4 Mar 11, 2026
c00e06a
Update generate_inputs to include release_version arg in README
kfang4 Mar 13, 2026
8f047b6
Revert to sample_id since replicate is appended
kfang4 Mar 19, 2026
4fef07e
Update wf-common
kfang4 Apr 22, 2026
5012fdd
Fix typo
kfang4 Apr 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
244 changes: 127 additions & 117 deletions README.md
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion wf-common
Open in desktop
Submodule wf-common updated 23 files
+16 −0 .gitignore
+401 −21 util/README.md
+0 −0 util/archive/transfer_raw_data
+398 −0 util/bucket_validation_utils.py
+201 −0 util/clean_wdl_raw_buckets
+142 −182 util/common.py
+807 −85 util/crn_cloud_collection_summary
+1 −1 util/data_promotion_diagram.svg
+150 −0 util/download_raw_bucket_metadata_to_local
+293 −0 util/generate_dataset_summary_table
+152 −44 util/generate_inputs
+335 −0 util/internal_qc_dataset_collection_summary
+5 −6 util/markdown_generator.py
+253 −0 util/promote_raw_data
+87 −84 util/promote_staging_data
+3 −1 util/requirements.txt
+0 −190 util/transfer_qc_metadata_and_contributed_data
+90 −28 util/transfer_qc_metadata_to_raw_bucket
+0 −102 util/transfer_raw_bucket_metadata_to_local
+161 −0 util/transfer_release_resources_to_raw_bucket.py
+31 −146 util/validate_raw_bucket_structure.py
+3 −3 wdl/structs.wdl
+0 −39 wdl/tasks/validate_workflow_name.wdl
8 changes: 5 additions & 3 deletions workflows/spatial_geomx/cohort_analysis/cohort_analysis.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ workflow cohort_analysis {
String workflow_name
String workflow_version
String workflow_release
String crn_release_version
String run_timestamp
String raw_data_path_prefix
Array[String] staging_data_buckets
Expand Down Expand Up @@ -94,7 +95,7 @@ workflow cohort_analysis {
input:
output_file_paths = preprocessing_output_file_paths,
staging_data_buckets = staging_data_buckets,
staging_data_path = "~{workflow_name}/preprocess",
staging_data_path = "~{workflow_name}/release/~{crn_release_version}/preprocess",
billing_project = billing_project,
zones = zones
}
Expand All @@ -103,7 +104,7 @@ workflow cohort_analysis {
input:
output_file_paths = processing_output_file_paths,
staging_data_buckets = staging_data_buckets,
staging_data_path = "~{workflow_name}/process_to_adata",
staging_data_path = "~{workflow_name}/release/~{crn_release_version}/process_to_adata",
billing_project = billing_project,
zones = zones
}
Expand Down Expand Up @@ -132,7 +133,7 @@ workflow cohort_analysis {
input:
output_file_paths = cohort_analysis_final_output_paths,
staging_data_buckets = staging_data_buckets,
staging_data_path = "~{workflow_name}/~{sub_workflow_name}",
staging_data_path = "~{workflow_name}/release/~{crn_release_version}/~{sub_workflow_name}",
billing_project = billing_project,
zones = zones
}
Expand Down Expand Up @@ -178,6 +179,7 @@ workflow cohort_analysis {
workflow_name: {help: "Workflow name; stored in the file-level manifest and final manifest with all saved files."}
workflow_version: {help: "Workflow version; stored in the file-level manifest and final manifest with all saved files."}
workflow_release: {help: "GitHub release; stored in the file-level manifest and final manifest with all saved files."}
crn_release_version: {help: "CRN Cloud release version; used to organize outputs and for the CRN Cloud release."}
run_timestamp: {help: "UTC timestamp; stored in the file-level manifest and final manifest with all saved files."}
raw_data_path_prefix: {help: "Raw data bucket path prefix; location of raw bucket to upload task outputs to (`<raw_data_bucket>/workflow_execution/cohort_analysis`)."}
staging_data_buckets: {help: "Array of staging data buckets to upload intermediate files to (i.e., DEV or UAT buckets depending on internal QC status)."}
Expand Down
2 changes: 1 addition & 1 deletion workflows/spatial_geomx/inputs.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"spatial_geomx_analysis.projects": "Array[WomCompositeType {\n slides -> Array[WomCompositeType {\n slide_id -> String\ngeomx_lab_annotation_xlsx -> File\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nsample_id -> String\nfastq_R1s -> Array[File]+\nbatch -> String?\nfastq_I2s -> Array[File]\nfastq_R2s -> Array[File]+ \n}] \n}]\nproject_sample_metadata_csv -> File\nteam_id -> String\ndataset_id -> String\ngeomx_config_ini -> File\nstaging_data_buckets -> Array[String]\nrun_project_cohort_analysis -> Boolean\ndataset_doi_url -> String\nraw_data_bucket -> String \n}]",
"spatial_geomx_analysis.projects": "Array[WomCompositeType {\n slides -> Array[WomCompositeType {\n asap_slide_id -> String\ngeomx_lab_annotation_xlsx -> File\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nfastq_R1s -> Array[File]+\nsample_id -> String\nbatch -> String?\nfastq_I2s -> Array[File]\nfastq_R2s -> Array[File]+ \n}] \n}]\ngeomx_config_ini -> File\nrun_project_cohort_analysis -> Boolean\nraw_data_bucket -> String\nasap_team_id -> String\nasap_dataset_id -> String\nasap_project_sample_metadata_csv -> File\nstaging_data_buckets -> Array[String]\nasap_dataset_doi_url -> String \n}]",
"spatial_geomx_analysis.geomxngs_config_pkc": "File",
"spatial_geomx_analysis.min_segment_reads": "Int (optional, default = 1000)",
"spatial_geomx_analysis.min_percent_reads_trimmed": "Int (optional, default = 80)",
Expand Down
14 changes: 8 additions & 6 deletions workflows/spatial_geomx/main.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ workflow spatial_geomx_analysis {
String workflow_name = "spatial_geomx"
String workflow_version = "v1.0.0"
String workflow_release = "https://github.com/ASAP-CRN/spatial-transcriptomics-wf/releases/tag/spatial_geomx_analysis-~{workflow_version}"
String crn_release_version = "v4.0.0"

@Alexander-Morin Alexander-Morin Apr 23, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same question as for the scRNA-seq pipeline, is this supposed to be hard-coded (and just updated per release?)

@kfang4 kfang4 Apr 24, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it's helpful to keep the release version and workflow version history in the GitHub repo. It's rare that the pipeline won't need an update when running new datasets, so usually the workflow_version and crn_release_version will both need to be bumped. Also, this is now the new way we'll be storing curated outputs-- in a crn_release_version folder instead of workflow_version folder.


call GetWorkflowMetadata.get_workflow_metadata {
input:
Expand All @@ -54,11 +55,11 @@ workflow spatial_geomx_analysis {

call Preprocess.preprocess {
input:
team_id = project.team_id,
dataset_id = project.dataset_id,
dataset_doi_url = project.dataset_doi_url,
team_id = project.asap_team_id,
dataset_id = project.asap_dataset_id,
dataset_doi_url = project.asap_dataset_doi_url,
slides = project.slides,
project_sample_metadata_csv = project.project_sample_metadata_csv,
project_sample_metadata_csv = project.asap_project_sample_metadata_csv,
geomx_config_ini = project.geomx_config_ini,
geomxngs_config_pkc = geomxngs_config_pkc,
min_segment_reads = min_segment_reads,
Expand Down Expand Up @@ -115,7 +116,7 @@ workflow spatial_geomx_analysis {
if (project.run_project_cohort_analysis) {
call CohortAnalysis.cohort_analysis as project_cohort_analysis {
input:
cohort_id = project.team_id,
cohort_id = project.asap_team_id,
project_sample_ids = preprocess.project_sample_ids,
processed_adata_objects = process_to_adata.processed_adata_objects,
preprocessing_output_file_paths = preprocessing_output_file_paths,
Expand All @@ -127,6 +128,7 @@ workflow spatial_geomx_analysis {
workflow_name = workflow_name,
workflow_version = workflow_version,
workflow_release = workflow_release,
crn_release_version = crn_release_version,
run_timestamp = get_workflow_metadata.timestamp,
raw_data_path_prefix = project_raw_data_path_prefix,
staging_data_buckets = project.staging_data_buckets,
Expand Down Expand Up @@ -182,7 +184,7 @@ workflow spatial_geomx_analysis {
}

meta {
description: "Harmonized human postmortem-derived brain sequencing (PMDBS) and non-human spatial transcriptomics workflow for Nanostring GeoMx data"
description: "Harmonized human postmortem-derived brain sequencing (PMDBS) and non-human spatial transcriptomics workflow for Nanostring GeoMx data."
}

parameter_meta {
Expand Down
26 changes: 13 additions & 13 deletions workflows/spatial_geomx/preprocess/preprocess.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ workflow preprocess {
String qc_raw_data_path = "~{workflow_raw_data_path_prefix}/qc/~{qc_task_version}"

scatter (slide_object in slides) {
String fastq_to_dcc_output = "~{dcc_raw_data_path}/~{slide_object.slide_id}.geomxngs_out_dir.tar.gz"
String dcc_to_rds_output = "~{rds_raw_data_path}/~{slide_object.slide_id}.NanoStringGeoMxSet.rds"
String qc_output = "~{qc_raw_data_path}/~{slide_object.slide_id}.qc.rds"
String fastq_to_dcc_output = "~{dcc_raw_data_path}/~{slide_object.asap_slide_id}.geomxngs_out_dir.tar.gz"
String dcc_to_rds_output = "~{rds_raw_data_path}/~{slide_object.asap_slide_id}.NanoStringGeoMxSet.rds"
String qc_output = "~{qc_raw_data_path}/~{slide_object.asap_slide_id}.qc.rds"
}

# For each sample, outputs an array of true/false: [fastq_to_dcc_complete, dcc_to_rds_complete, qc_complete]
Expand All @@ -78,13 +78,13 @@ workflow preprocess {
String dcc_to_rds_complete = check_output_files_exist.sample_preprocessing_complete[slide_index][1]
String qc_complete = check_output_files_exist.sample_preprocessing_complete[slide_index][2]

String fastq_to_dcc_geomxngs_dcc_zip = "~{dcc_raw_data_path}/~{slide.slide_id}.DCC.zip"
String fastq_to_dcc_geomxngs_output_tar_gz = "~{dcc_raw_data_path}/~{slide.slide_id}.geomxngs_out_dir.tar.gz"
String fastq_to_dcc_geomxngs_dcc_zip = "~{dcc_raw_data_path}/~{slide.asap_slide_id}.DCC.zip"
String fastq_to_dcc_geomxngs_output_tar_gz = "~{dcc_raw_data_path}/~{slide.asap_slide_id}.geomxngs_out_dir.tar.gz"

if (fastq_to_dcc_complete == "false") {
call fastq_to_dcc {
input:
slide_id = slide.slide_id,
slide_id = slide.asap_slide_id,
fastq_R1s = flatten(fastq_R1s),
fastq_R2s = flatten(fastq_R2s),
geomx_config_ini = geomx_config_ini,
Expand All @@ -99,14 +99,14 @@ workflow preprocess {
File geomxngs_dcc_zip_output = select_first([fastq_to_dcc.geomxngs_dcc_zip, fastq_to_dcc_geomxngs_dcc_zip]) #!FileCoercion
File geomxngs_output_tar_gz_output = select_first([fastq_to_dcc.geomxngs_output_tar_gz, fastq_to_dcc_geomxngs_output_tar_gz]) #!FileCoercion

String dcc_to_rds_object = "~{rds_raw_data_path}/~{slide.slide_id}.NanoStringGeoMxSet.rds"
String dcc_to_rds_object = "~{rds_raw_data_path}/~{slide.asap_slide_id}.NanoStringGeoMxSet.rds"

if (dcc_to_rds_complete == "false") {
call dcc_to_rds {
input:
team_id = team_id,
dataset_id = dataset_id,
slide_id = slide.slide_id,
slide_id = slide.asap_slide_id,
project_sample_metadata_csv = project_sample_metadata_csv,
geomxngs_dcc_zip = geomxngs_dcc_zip_output,
geomx_lab_annotation_xlsx = slide.geomx_lab_annotation_xlsx,
Expand All @@ -121,15 +121,15 @@ workflow preprocess {

File initial_rds_object_output = select_first([dcc_to_rds.initial_rds_object, dcc_to_rds_object]) #!FileCoercion

String qc_metrics_rds_object = "~{qc_raw_data_path}/~{slide.slide_id}.qc.rds"
String qc_segment_summary_csv = "~{qc_raw_data_path}/~{slide.slide_id}.segment_qc_summary.csv"
String qc_probe_summary_csv = "~{qc_raw_data_path}/~{slide.slide_id}.probe_qc_summary.csv"
String qc_gene_count_csv = "~{qc_raw_data_path}/~{slide.slide_id}.gene_count.csv"
String qc_metrics_rds_object = "~{qc_raw_data_path}/~{slide.asap_slide_id}.qc.rds"
String qc_segment_summary_csv = "~{qc_raw_data_path}/~{slide.asap_slide_id}.segment_qc_summary.csv"
String qc_probe_summary_csv = "~{qc_raw_data_path}/~{slide.asap_slide_id}.probe_qc_summary.csv"
String qc_gene_count_csv = "~{qc_raw_data_path}/~{slide.asap_slide_id}.gene_count.csv"

if (qc_complete == "false") {
call qc {
input:
slide_id = slide.slide_id,
slide_id = slide.asap_slide_id,
initial_rds_object = initial_rds_object_output,
min_segment_reads = min_segment_reads,
min_percent_reads_trimmed = min_percent_reads_trimmed,
Expand Down
10 changes: 5 additions & 5 deletions workflows/spatial_geomx/structs.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,19 @@ struct Sample {
}

struct Slide {
String slide_id
String asap_slide_id
File geomx_lab_annotation_xlsx

Array[Sample] samples
}

struct Project {
String team_id
String dataset_id
String dataset_doi_url
String asap_team_id
String asap_dataset_id
String asap_dataset_doi_url
Array[Slide] slides

File project_sample_metadata_csv
File asap_project_sample_metadata_csv
File geomx_config_ini

Boolean run_project_cohort_analysis
Expand Down
6 changes: 4 additions & 2 deletions workflows/spatial_visium/cohort_analysis/cohort_analysis.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ workflow cohort_analysis {
String workflow_name
String workflow_version
String workflow_release
String crn_release_version
String run_timestamp
String raw_data_path_prefix
Array[String] staging_data_buckets
Expand Down Expand Up @@ -126,7 +127,7 @@ workflow cohort_analysis {
input:
output_file_paths = preprocessing_output_file_paths,
staging_data_buckets = staging_data_buckets,
staging_data_path = "~{workflow_name}/preprocess",
staging_data_path = "~{workflow_name}/release/~{crn_release_version}/preprocess",
billing_project = billing_project,
zones = zones
}
Expand Down Expand Up @@ -163,7 +164,7 @@ workflow cohort_analysis {
input:
output_file_paths = cohort_analysis_final_output_paths,
staging_data_buckets = staging_data_buckets,
staging_data_path = "~{workflow_name}/~{sub_workflow_name}",
staging_data_path = "~{workflow_name}/release/~{crn_release_version}/~{sub_workflow_name}",
billing_project = billing_project,
zones = zones
}
Expand Down Expand Up @@ -221,6 +222,7 @@ workflow cohort_analysis {
workflow_name: {help: "Workflow name; stored in the file-level manifest and final manifest with all saved files."}
workflow_version: {help: "Workflow version; stored in the file-level manifest and final manifest with all saved files."}
workflow_release: {help: "GitHub release; stored in the file-level manifest and final manifest with all saved files."}
crn_release_version: {help: "CRN Cloud release version; used to organize outputs and for the CRN Cloud release."}
run_timestamp: {help: "UTC timestamp; stored in the file-level manifest and final manifest with all saved files."}
raw_data_path_prefix: {help: "Raw data bucket path prefix; location of raw bucket to upload task outputs to (`<raw_data_bucket>/workflow_execution/cohort_analysis`)."}
staging_data_buckets: {help: "Array of staging data buckets to upload intermediate files to (i.e., DEV or UAT buckets depending on internal QC status)."}
Expand Down
2 changes: 1 addition & 1 deletion workflows/spatial_visium/inputs.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"spatial_visium_analysis.projects": "Array[WomCompositeType {\n team_id -> String\ndataset_id -> String\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nsample_id -> String\nfastq_R1s -> Array[File]+\nbatch -> String?\nvisium_capture_area -> String\nvisium_brightfield_image -> File\nfastq_I2s -> Array[File]\nvisium_slide_serial_number -> String\nfastq_R2s -> Array[File]+ \n}]\nstaging_data_buckets -> Array[String]\nrun_project_cohort_analysis -> Boolean\ndataset_doi_url -> String\nraw_data_bucket -> String \n}]",
"spatial_visium_analysis.projects": "Array[WomCompositeType {\n samples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nfastq_R1s -> Array[File]+\nsample_id -> String\nbatch -> String?\nvisium_capture_area -> String\nvisium_brightfield_image -> File\nfastq_I2s -> Array[File]\nvisium_slide_serial_number -> String\nfastq_R2s -> Array[File]+ \n}]\nasap_dataset_id -> String\nrun_project_cohort_analysis -> Boolean\nraw_data_bucket -> String\nasap_team_id -> String\nstaging_data_buckets -> Array[String]\nasap_dataset_doi_url -> String \n}]",
"spatial_visium_analysis.spaceranger_reference_data": "File",
"spatial_visium_analysis.visium_probe_set_csv": "File? (optional)",
"spatial_visium_analysis.filter_cells_min_counts": "Int (optional, default = 5000)",
Expand Down
12 changes: 7 additions & 5 deletions workflows/spatial_visium/main.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ workflow spatial_visium_analysis {
String workflow_name = "spatial_visium"
String workflow_version = "v1.0.1"
String workflow_release = "https://github.com/ASAP-CRN/spatial-transcriptomics-wf/releases/tag/spatial_visium_analysis-~{workflow_version}"
String crn_release_version = "v4.0.0"

call GetWorkflowMetadata.get_workflow_metadata {
input:
Expand All @@ -44,9 +45,9 @@ workflow spatial_visium_analysis {

call Preprocess.preprocess {
input:
team_id = project.team_id,
dataset_id = project.dataset_id,
dataset_doi_url = project.dataset_doi_url,
team_id = project.asap_team_id,
dataset_id = project.asap_dataset_id,
dataset_doi_url = project.asap_dataset_doi_url,
samples = project.samples,
spaceranger_reference_data = spaceranger_reference_data,
visium_probe_set_csv = visium_probe_set_csv,
Expand Down Expand Up @@ -77,7 +78,7 @@ workflow spatial_visium_analysis {
if (project.run_project_cohort_analysis) {
call CohortAnalysis.cohort_analysis as project_cohort_analysis {
input:
cohort_id = project.team_id,
cohort_id = project.asap_team_id,
project_sample_ids = preprocess.project_sample_ids,
preprocessed_adata_objects = preprocess.qc_adata_object,
preprocessing_output_file_paths = preprocessing_output_file_paths,
Expand All @@ -93,6 +94,7 @@ workflow spatial_visium_analysis {
workflow_name = workflow_name,
workflow_version = workflow_version,
workflow_release = workflow_release,
crn_release_version = crn_release_version,
run_timestamp = get_workflow_metadata.timestamp,
raw_data_path_prefix = project_raw_data_path_prefix,
staging_data_buckets = project.staging_data_buckets,
Expand Down Expand Up @@ -151,7 +153,7 @@ workflow spatial_visium_analysis {
}

meta {
description: "Harmonized human postmortem-derived brain sequencing (PMDBS) and non-human spatial transcriptomics workflow for 10x Visium data"
description: "Harmonized human postmortem-derived brain sequencing (PMDBS) and non-human spatial transcriptomics workflow for 10x Visium data."
}

parameter_meta {
Expand Down
6 changes: 3 additions & 3 deletions workflows/spatial_visium/structs.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ struct Sample {
}

struct Project {
String team_id
String dataset_id
String dataset_doi_url
String asap_team_id
String asap_dataset_id
String asap_dataset_doi_url
Array[Sample] samples

Boolean run_project_cohort_analysis
Expand Down
Loading