From 9f38fbbec588bc01bc61bde65eae358ef5050211 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 17 Sep 2025 16:15:30 +0200 Subject: [PATCH 1/9] update paths in workflow file --- .github/workflows/test-mars.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index 19015dd..07b72e1 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -2,10 +2,10 @@ name: Test MARS on: push: paths: - - "mars-cli/**" + - "mars_lib/**" pull_request: paths: - - "mars-cli/**" + - "mars_lib/**" branches: - main workflow_dispatch: @@ -17,8 +17,6 @@ jobs: os: [ubuntu-latest, windows-latest] python-version: ["3.9", "3.13"] runs-on: ${{ matrix.os }} - env: - working-directory: ./mars-cli steps: - uses: actions/checkout@v4 From d56ec44bd806c77d6ecf9a48e2aa46d4f56c747d Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 17 Sep 2025 16:31:07 +0200 Subject: [PATCH 2/9] Do sparse-checkout for the test data --- .github/workflows/test-mars.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index 07b72e1..20d0411 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -20,6 +20,11 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: https://github.com/elixir-europe/MARS.git + path: . + sparse-checkout: 'test-data/' - name: Set up Python uses: actions/setup-python@v5 with: From 479af37012aeb9a42f985176d0f917c90a5a3ac0 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 17 Sep 2025 16:34:54 +0200 Subject: [PATCH 3/9] Change checkout version --- .github/workflows/test-mars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index 20d0411..9f0e69e 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -19,10 +19,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + - uses: actions/checkout@v5 with: - repository: https://github.com/elixir-europe/MARS.git + repository: https://github.com/elixir-europe/MARS path: . sparse-checkout: 'test-data/' - name: Set up Python From 2b4c4c2bac6fff5ad6667f10a04ff5f8d8ccdd9e Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 17 Sep 2025 16:36:18 +0200 Subject: [PATCH 4/9] Correct repo format --- .github/workflows/test-mars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index 9f0e69e..658cc3f 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v5 - uses: actions/checkout@v5 with: - repository: https://github.com/elixir-europe/MARS + repository: elixir-europe/MARS path: . sparse-checkout: 'test-data/' - name: Set up Python From 8bf5651d2b0c34b49d99d7a7fdf9b6c081c73fd7 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 17 Sep 2025 16:37:59 +0200 Subject: [PATCH 5/9] update cache dependency path --- .github/workflows/test-mars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index 658cc3f..0c15e07 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -30,7 +30,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: '**/setup.py' + cache-dependency-path: 'setup.py' - run: | pip install -e '.[test]' pip install keyring keyrings.alt From 32311fb2a75f27d849313813116499c97aae7214 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 17 Sep 2025 16:51:46 +0200 Subject: [PATCH 6/9] one more try... --- .github/workflows/test-mars.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index 0c15e07..a4cd882 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -23,14 +23,16 @@ jobs: - uses: actions/checkout@v5 with: repository: elixir-europe/MARS - path: . - sparse-checkout: 'test-data/' + path: test-data + sparse-checkout: 'test-data/*' + - name: show files + run: ls -la . - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: 'setup.py' + cache-dependency-path: '**/setup.py' - run: | pip install -e '.[test]' pip install keyring keyrings.alt From 873a62795c63840b5a8a0e3da1c35bf52e8259df Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 17 Sep 2025 16:52:53 +0200 Subject: [PATCH 7/9] one more try... --- .github/workflows/test-mars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index a4cd882..84c6f13 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -24,9 +24,9 @@ jobs: with: repository: elixir-europe/MARS path: test-data - sparse-checkout: 'test-data/*' + sparse-checkout: 'test-data/' - name: show files - run: ls -la . + run: ls -la test-data - name: Set up Python uses: actions/setup-python@v5 with: From ff1b0a526e0fbad46c273009c2c25831a4d8c72d Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Thu, 18 Sep 2025 09:48:36 +0200 Subject: [PATCH 8/9] Add MARS repo as submodule for test data --- .github/workflows/test-mars.yml | 5 +- .gitmodules | 3 + MARS | 1 + tests/test_biosample_external_references.py | 2 +- tests/test_ftp_upload.py | 2 +- tests/test_isa_json.py | 68 ++++++++++----------- 6 files changed, 41 insertions(+), 40 deletions(-) create mode 100644 .gitmodules create mode 160000 MARS diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index 84c6f13..835ce17 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -19,12 +19,9 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v5 - uses: actions/checkout@v5 with: - repository: elixir-europe/MARS - path: test-data - sparse-checkout: 'test-data/' + submodules: 'recursive' - name: show files run: ls -la test-data - name: Set up Python diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2fd802d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "MARS"] + path = MARS + url = https://github.com/elixir-europe/MARS.git diff --git a/MARS b/MARS new file mode 160000 index 0000000..ed97522 --- /dev/null +++ b/MARS @@ -0,0 +1 @@ +Subproject commit ed975227017f592650d9fa700a35f33bd672fcda diff --git a/tests/test_biosample_external_references.py b/tests/test_biosample_external_references.py index ab325d8..6dc6b79 100644 --- a/tests/test_biosample_external_references.py +++ b/tests/test_biosample_external_references.py @@ -36,7 +36,7 @@ def test_load_json_file(): def test_handle_input_dict(): - input = "../test-data/ISA-BH2023-ALL/isa-bh2023-all.json" + input = "MARS/test-data/ISA-BH2023-ALL/isa-bh2023-all.json" assert handle_input_dict(input) not_a_json = "./tests/fixtures/bad_json.json" diff --git a/tests/test_ftp_upload.py b/tests/test_ftp_upload.py index a5d1672..227b158 100644 --- a/tests/test_ftp_upload.py +++ b/tests/test_ftp_upload.py @@ -23,7 +23,7 @@ def test_upload_success(): uploader = FTPUploader("webin2.ebi.ac.uk", creds["username"], creds["password"]) uploader.upload( [ - Path("../test-data/ENA_TEST2.R1.fastq.gz"), + Path("MARS/test-data/ENA_TEST2.R1.fastq.gz"), Path("./tests/fixtures/not_a_json_file.txt"), ] ) diff --git a/tests/test_isa_json.py b/tests/test_isa_json.py index a50517b..b358c63 100644 --- a/tests/test_isa_json.py +++ b/tests/test_isa_json.py @@ -27,13 +27,13 @@ def test_load_isa_json(): # Should test the validation process of the ISA JSON file where root level = investigation. valid_isa_json01 = load_isa_json( - "../test-data/ISA-BH2023-ALL/isa-bh2023-all.json", True + "MARS/test-data/ISA-BH2023-ALL/isa-bh2023-all.json", True ) assert len(valid_isa_json01.investigation.studies) == 1 assert valid_isa_json01.investigation.studies[0].identifier == "BH2023" # Should test the validation process of the ISA JSON file where root has 'investigation' as key. - valid_isa_json02 = load_isa_json("../test-data/biosamples-input-isa.json", False) + valid_isa_json02 = load_isa_json("MARS/test-data/biosamples-input-isa.json", False) assert len(valid_isa_json02.investigation.studies) == 1 assert valid_isa_json02.investigation.studies[0].title == "Arabidopsis thaliana" @@ -43,7 +43,7 @@ def test_load_isa_json(): def test_reduce_isa_json_for_target_repo(): good_isa_json = load_isa_json( - "../test-data/ISA-BH2023-ALL/isa-bh2023-all.json", True + "MARS/test-data/ISA-BH2023-ALL/isa-bh2023-all.json", True ) filtered_isa_json = reduce_isa_json_for_target_repo( @@ -60,7 +60,7 @@ def test_reduce_isa_json_for_target_repo(): def test_reduce_isa_json_for_biosamples(): good_isa_json = load_isa_json( - "../test-data/ISA-BH2023-ALL/isa-bh2023-all.json", True + "MARS/test-data/ISA-BH2023-ALL/isa-bh2023-all.json", True ) filtered_isa_json = reduce_isa_json_for_target_repo( @@ -180,7 +180,7 @@ def test_person_phone_nr_validator(): def test_update_study_materials_no_accession_categories(): # This file has no characteristics for accessions - json_path = "../test-data/biosamples-original-isa-no-accesion-char.json" + json_path = "MARS/test-data/biosamples-original-isa-no-accesion-char.json" with open(json_path) as json_file: json_data = json.load(json_file) @@ -214,7 +214,7 @@ def test_update_study_materials_no_accession_categories(): def test_update_study_materials_with_accession_categories(): # This file has no characteristics for accessions - json_path = "../test-data/biosamples-original-isa.json" + json_path = "MARS/test-data/biosamples-original-isa.json" with open(json_path) as json_file: json_data = json.load(json_file) @@ -308,45 +308,45 @@ def test_filename_validation(): def test_map_data_files_to_repositories(): isa_json = load_isa_json( - file_path="../test-data/ISA-BH2024-ALL/isa-bh2024-all.json", + file_path="MARS/test-data/ISA-BH2024-ALL/isa-bh2024-all.json", investigation_is_root=True, ) exact_match_files = [ - "../test-data/ISA-BH2024-ALL/cnv-seq-data-0.fastq", - "../test-data/ISA-BH2024-ALL/cnv-seq-data-1.fastq", - "../test-data/ISA-BH2024-ALL/cnv-seq-data-2.fastq", - "../test-data/ISA-BH2024-ALL/cnv-seq-data-3.fastq", - "../test-data/ISA-BH2024-ALL/metpro-analysis.txt", - "../test-data/ISA-BH2024-ALL/ms-data-metpro--1.mzml", - "../test-data/ISA-BH2024-ALL/ms-data-metpro--2.mzml", - "../test-data/ISA-BH2024-ALL/ms-data-metpro--3.mzml", - "../test-data/ISA-BH2024-ALL/ms-data-metpro--4.mzml", - "../test-data/ISA-BH2024-ALL/rna-seq-data-0.fastq", - "../test-data/ISA-BH2024-ALL/rna-seq-data-1.fastq", - "../test-data/ISA-BH2024-ALL/rna-seq-data-2.fastq", - "../test-data/ISA-BH2024-ALL/rna-seq-data-3.fastq", + "MARS/test-data/ISA-BH2024-ALL/cnv-seq-data-0.fastq", + "MARS/test-data/ISA-BH2024-ALL/cnv-seq-data-1.fastq", + "MARS/test-data/ISA-BH2024-ALL/cnv-seq-data-2.fastq", + "MARS/test-data/ISA-BH2024-ALL/cnv-seq-data-3.fastq", + "MARS/test-data/ISA-BH2024-ALL/metpro-analysis.txt", + "MARS/test-data/ISA-BH2024-ALL/ms-data-metpro--1.mzml", + "MARS/test-data/ISA-BH2024-ALL/ms-data-metpro--2.mzml", + "MARS/test-data/ISA-BH2024-ALL/ms-data-metpro--3.mzml", + "MARS/test-data/ISA-BH2024-ALL/ms-data-metpro--4.mzml", + "MARS/test-data/ISA-BH2024-ALL/rna-seq-data-0.fastq", + "MARS/test-data/ISA-BH2024-ALL/rna-seq-data-1.fastq", + "MARS/test-data/ISA-BH2024-ALL/rna-seq-data-2.fastq", + "MARS/test-data/ISA-BH2024-ALL/rna-seq-data-3.fastq", ] check_map = dict( { "metabolights": [ - "../test-data/ISA-BH2024-ALL/metpro-analysis.txt", - "../test-data/ISA-BH2024-ALL/ms-data-metpro--1.mzml", - "../test-data/ISA-BH2024-ALL/ms-data-metpro--2.mzml", - "../test-data/ISA-BH2024-ALL/ms-data-metpro--3.mzml", - "../test-data/ISA-BH2024-ALL/ms-data-metpro--4.mzml", + "MARS/test-data/ISA-BH2024-ALL/metpro-analysis.txt", + "MARS/test-data/ISA-BH2024-ALL/ms-data-metpro--1.mzml", + "MARS/test-data/ISA-BH2024-ALL/ms-data-metpro--2.mzml", + "MARS/test-data/ISA-BH2024-ALL/ms-data-metpro--3.mzml", + "MARS/test-data/ISA-BH2024-ALL/ms-data-metpro--4.mzml", ], "arrayexpress": [ - "../test-data/ISA-BH2024-ALL/rna-seq-data-0.fastq", - "../test-data/ISA-BH2024-ALL/rna-seq-data-1.fastq", - "../test-data/ISA-BH2024-ALL/rna-seq-data-2.fastq", - "../test-data/ISA-BH2024-ALL/rna-seq-data-3.fastq", + "MARS/test-data/ISA-BH2024-ALL/rna-seq-data-0.fastq", + "MARS/test-data/ISA-BH2024-ALL/rna-seq-data-1.fastq", + "MARS/test-data/ISA-BH2024-ALL/rna-seq-data-2.fastq", + "MARS/test-data/ISA-BH2024-ALL/rna-seq-data-3.fastq", ], "eva": [ - "../test-data/ISA-BH2024-ALL/cnv-seq-data-0.fastq", - "../test-data/ISA-BH2024-ALL/cnv-seq-data-1.fastq", - "../test-data/ISA-BH2024-ALL/cnv-seq-data-2.fastq", - "../test-data/ISA-BH2024-ALL/cnv-seq-data-3.fastq", + "MARS/test-data/ISA-BH2024-ALL/cnv-seq-data-0.fastq", + "MARS/test-data/ISA-BH2024-ALL/cnv-seq-data-1.fastq", + "MARS/test-data/ISA-BH2024-ALL/cnv-seq-data-2.fastq", + "MARS/test-data/ISA-BH2024-ALL/cnv-seq-data-3.fastq", ], } ) @@ -361,7 +361,7 @@ def test_map_data_files_to_repositories(): map_data_files_to_repositories(not_enough_files, isa_json) too_many_files = exact_match_files.copy() - one_too_many = "../test-data/ISA-BH2024-ALL/one-too-many.fastq" + one_too_many = "MARS/test-data/ISA-BH2024-ALL/one-too-many.fastq" too_many_files.append(one_too_many) result_maps = map_data_files_to_repositories(too_many_files, isa_json) From a9528881f24fa74514f25c08e7959680cb791e9c Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Thu, 18 Sep 2025 09:53:29 +0200 Subject: [PATCH 9/9] Remove ls --- .github/workflows/test-mars.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-mars.yml b/.github/workflows/test-mars.yml index 835ce17..150d3d7 100644 --- a/.github/workflows/test-mars.yml +++ b/.github/workflows/test-mars.yml @@ -22,8 +22,6 @@ jobs: - uses: actions/checkout@v5 with: submodules: 'recursive' - - name: show files - run: ls -la test-data - name: Set up Python uses: actions/setup-python@v5 with: