From 0296a918380249f8b77a8d58f357c756267dfdf2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 6 Oct 2025 19:39:32 +0200 Subject: [PATCH 01/34] use consistent config for 1.4.5/6/7/8 and reorganize entries --- notebooks/cosmo_val/cat_config.yaml | 197 ++++++++++++++++++++-------- 1 file changed, 141 insertions(+), 56 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 13991a4..7893d92 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -7,6 +7,49 @@ nz: path: dndz blind: A +SP_test: + pipeline: SP + subdir: CFIS/v1.0/ShapePipe/ + ls: dashed + colour: black + marker: d + getdist_colour: 0.0, 0.5, 1.0 + cov_th: + A: 2420.2651014497287 #deg^2 + n_e: 7.040818382014773 #arcmin-2 + n_psf: 0.752316232272063 #arcmin-2 + sigma_e: 0.30961528707207325 + shear: + path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + e1_col: e1 + e2_col: e2 + e1_PSF_col: e1_PSF + e2_PSF_col: e2_PSF + w_col: w_iv + R: 1.0 + redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + star: + path: unions_shapepipe_star_2024_v1.4.1.fits + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + psf: + path: unions_shapepipe_psf_2024_v1.4.1.fits + hdu: 1 + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e2_PSF_col: E2_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_star_col: E2_STAR_HSM + PSF_size: SIGMA_PSF_HSM + star_size: SIGMA_STAR_HSM + PSF_flag: "FLAG_PSF_HSM" + star_flag: "FLAG_STAR_HSM" + square_size: True + SP_v1.0: pipeline: SP subdir: CFIS/v1.0/ShapePipe @@ -722,36 +765,36 @@ SP_v1.4.5: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.4.6: +SP_v1.4.5_glass_mock: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: CFIS/v1.0/ShapePipe ls: dashed - colour: green + colour: crimson marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 + A: 2782 #deg^2 + n_e: 7.18 #arcmin-2 n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 + sigma_e: 0.4370966656902571 shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: unions_v1.4.5_glass_mock.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_des + w_col: w R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.a.fits + path: unions_shapepipe_star_2024_v1.4.1.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.1.fits hdu: 1 ra_col: RA dec_col: Dec @@ -764,27 +807,27 @@ SP_v1.4.6: PSF_flag: "FLAG_PSF_HSM" star_flag: "FLAG_STAR_HSM" square_size: True - -SP_v1.4.5.A: + +SP_v1.4.5_leak_corr: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 ls: dashed - colour: red + colour: teal marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 + A: 2782 #deg^2 + n_e: 7.18 #arcmin-2 n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 + sigma_e: 0.4370966656902571 shear: - path: ../../../../../../guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5/shapepipe_SPv1.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: g1 - e2_col: g2 + e1_col: e1_leak_corrected + e2_col: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w + w_col: w_des R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: @@ -794,7 +837,7 @@ SP_v1.4.5.A: e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -937,37 +980,36 @@ SP_v1.4.5_intermediate: star_flag: "FLAG_STAR_HSM" square_size: True - -SP_v1.4.5_glass_mock: +SP_v1.4.5.A: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: CFIS/v1.0/ShapePipe/ ls: dashed - colour: crimson + colour: red marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 + A: 2420.2651014497287 #deg^2 + n_e: 7.040818382014773 #arcmin-2 n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 + sigma_e: 0.30961528707207325 shear: - path: unions_v1.4.5_glass_mock.fits + path: ../../../../../../guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5/shapepipe_SPv1.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 + e1_col: g1 + e2_col: g2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w_col: w + w: w R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.1.fits + path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -980,37 +1022,37 @@ SP_v1.4.5_glass_mock: PSF_flag: "FLAG_PSF_HSM" star_flag: "FLAG_STAR_HSM" square_size: True - -SP_test: + +SP_v1.4.6: pipeline: SP subdir: CFIS/v1.0/ShapePipe/ ls: dashed - colour: black + colour: green marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 + A: 2420.2651014497287 #deg^2 - shouldn't change with size cuts + n_e: null # TBD + n_psf: null # TBD + sigma_e: null # TBD shear: - path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w_col: w_iv + w_col: w_des R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.1.fits + path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1024,20 +1066,20 @@ SP_test: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.4.5_leak_corr: +SP_v1.4.6_leak_corr: pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 + subdir: CFIS/v1.0/ShapePipe/ ls: dashed - colour: teal + colour: green marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 + A: 2420.2651014497287 #deg^2 - shouldn't change with size cuts + n_e: null # TBD + n_psf: null # TBD + sigma_e: null # TBD shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5.fits + path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1_leak_corrected e2_col: e2_leak_corrected @@ -1047,13 +1089,56 @@ SP_v1.4.5_leak_corr: R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.a.fits + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e2_PSF_col: E2_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_star_col: E2_STAR_HSM + PSF_size: SIGMA_PSF_HSM + star_size: SIGMA_STAR_HSM + PSF_flag: "FLAG_PSF_HSM" + star_flag: "FLAG_STAR_HSM" + square_size: True + +SP_v1.4.7_leak_corr: + pipeline: SP + subdir: CFIS/v1.0/ShapePipe/ + ls: dashed + colour: darkorchid + marker: d + getdist_colour: 0.0, 0.5, 1.0 + cov_th: + A: null # TBD + n_e: null # TBD + n_psf: null # TBD + sigma_e: null # TBD + shear: + path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + e1_col: e1_leak_corrected + e2_col: e2_leak_corrected + e1_PSF_col: e1_PSF + e2_PSF_col: e2_PSF + w_col: w_des + R: 1.0 + redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + star: + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + psf: + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec From 8f9aab0fcf41b30e342fd10a808b56a5b0b736de Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 6 Oct 2025 19:40:38 +0200 Subject: [PATCH 02/34] add 1.4.8 --- notebooks/cosmo_val/cat_config.yaml | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 7893d92..88c3068 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1152,6 +1152,49 @@ SP_v1.4.7_leak_corr: star_flag: "FLAG_STAR_HSM" square_size: True +SP_v1.4.8_leak_corr: + pipeline: SP + subdir: CFIS/v1.0/ShapePipe/ + ls: dashed + colour: darkorchid + marker: d + getdist_colour: 0.0, 0.5, 1.0 + cov_th: + A: null # TBD + n_e: null # TBD + n_psf: null # TBD + sigma_e: null # TBD + shear: + path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + e1_col: e1_leak_corrected + e2_col: e2_leak_corrected + e1_PSF_col: e1_PSF + e2_PSF_col: e2_PSF + w_col: w_des + R: 1.0 + redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + star: + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + psf: + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e2_PSF_col: E2_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_star_col: E2_STAR_HSM + PSF_size: SIGMA_PSF_HSM + star_size: SIGMA_STAR_HSM + PSF_flag: "FLAG_PSF_HSM" + star_flag: "FLAG_STAR_HSM" + square_size: True + SP_v1.5.4: pipeline: SP subdir: ../../../UNIONS/WL/v1.5.x From 1b3f5397a4a322fc5844ea55e78ecbcbfc71ef1b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Oct 2025 00:12:51 +0200 Subject: [PATCH 03/34] refactor: standardise catalog paths and add guardrails --- docs/PR_DRAFT.md | 38 ++ notebooks/cosmo_val/cat_config.yaml | 366 +++++++----------- notebooks/cosmo_val/run_cosmo_val.py | 8 +- src/sp_validation/cosmo_val.py | 80 +++- src/sp_validation/tests/test_catalog_paths.py | 97 +++++ src/sp_validation/tests/test_cosmo_val.py | 127 ++++++ 6 files changed, 474 insertions(+), 242 deletions(-) create mode 100644 docs/PR_DRAFT.md create mode 100644 src/sp_validation/tests/test_catalog_paths.py create mode 100644 src/sp_validation/tests/test_cosmo_val.py diff --git a/docs/PR_DRAFT.md b/docs/PR_DRAFT.md new file mode 100644 index 0000000..38b1bfe --- /dev/null +++ b/docs/PR_DRAFT.md @@ -0,0 +1,38 @@ +# Draft PR: Catalog Path Cleanup & Guardrails + +## Summary + +- Normalised every `subdir` entry in `notebooks/cosmo_val/cat_config.yaml` so the loader relies on explicit `/n17data/...` locations rather than the old `data_dir` fallback. Shared resources (redshift distributions, common star/psf FITS) now point at their canonical mounts. +- Updated the Pseudo-`C_\ell` code path in `src/sp_validation/cosmo_val.py` to read redshift files directly from the catalog configuration, removing the `data_base_dir` assumption. +- Added `src/sp_validation/tests/test_catalog_paths.py`, a lightweight validator that resolves each shear/star/psf reference and records a small allow-list for catalogues whose source files are currently unavailable. +- Kept the additive-bias regression test focused on the catalogues that still have data on disk; the missing releases stay excluded so CI remains green. + +## Outstanding Data Gaps + +All of the catalogues below are missing their **shear** FITS on the shared mounts (star/psf often missing as well): + +1. `SP_v1.0`, `SP_v1.1`, and `SP_matched_MP_v1.0` +2. The `SP_v1.4` family: `SP_v1.4`, `SP_v1.4_conv`, `SP_v1.4_noalpha` +3. Every `SP_v1.4-P1+3*` derivative (same missing 2022-era inputs as item 2) + +These entries remain in `cat_config.yaml` for provenance, are listed in the validator allow-list, and are intentionally omitted from the additive-bias parameter set. The follow-up PR should either restore their data to `/n17data/...`, retarget them at new releases (e.g. the 2024 v1.4.1 products), or mark them as deprecated. + +## Proposed Next Steps + +1. **Decide The Fate Of The Legacy Catalogues** + - Recover the missing FITS/parquet files, or formally deprecate the versions above (update docs/tests accordingly). + +2. **Re-home The v1.4 Variants (if data exist)** + - Point each affected block at the correct `/n17data/UNIONS/WL/v1.4.x//` directory. + - Prefer catalogue-local star/psf fits when the directory is more than a symlink; fall back to the shared root otherwise. + - Remove the entries from `EXPECTED_MISSING` and re-enable them in regression tests once their files resolve. + +3. **Document The Conventions** + - Add a short developer note describing the absolute-path policy, preferred star/psf strategy, and how to run the new path validator before pushing catalog edits. + - Optionally list the currently offline datasets so collaborators know what needs recovering. + +## Validation Checklist + +- `app python -m pytest src/sp_validation/tests/test_catalog_paths.py` +- `app python -m pytest src/sp_validation/tests/test_cosmo_val.py -k additive_bias` +- Manual `ls` spot-check of any retargeted catalogue directories to ensure we pick up the intended local star/psf files rather than parent symlinks. diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 88c3068..493b5db 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -2,14 +2,16 @@ paths: output: ./output nz: - subdir: CFIS/v1.0/nz + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz + dndz: path: dndz blind: A SP_test: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: dashed colour: black marker: d @@ -28,7 +30,7 @@ SP_test: e2_PSF_col: e2_PSF w_col: w_iv R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: path: unions_shapepipe_star_2024_v1.4.1.fits ra_col: RA @@ -52,7 +54,8 @@ SP_test: SP_v1.0: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: blue marker: o @@ -87,7 +90,8 @@ SP_v1.0: SP_v1.1: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: dashed colour: b marker: s @@ -120,7 +124,8 @@ SP_v1.1: SP_v1.3: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: green marker: p @@ -158,7 +163,8 @@ SP_v1.3: SP_v1.4: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: brown marker: d @@ -205,7 +211,8 @@ SP_v1.4: SP_v1.4_conv: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: black marker: d @@ -252,7 +259,8 @@ SP_v1.4_conv: SP_v1.4_noalpha: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: brown marker: d @@ -300,7 +308,8 @@ SP_v1.4_noalpha: SP_v1.4-P3: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 + ls: dashed colour: brown marker: d @@ -346,7 +355,8 @@ SP_v1.4-P3: SP_v1.4-P1+3: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + ls: dashed colour: brown marker: d @@ -390,7 +400,8 @@ SP_v1.4-P1+3: SP_v1.4-P1+3_wcs: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + ls: dashed colour: brown marker: d @@ -432,7 +443,8 @@ SP_v1.4-P1+3_wcs: SP_v1.4-P1+3_no_alpha: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + ls: dashed colour: black marker: d @@ -474,7 +486,8 @@ SP_v1.4-P1+3_no_alpha: SP_v1.4-P1+3_li_2024: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + ls: dashed colour: gold marker: d @@ -516,7 +529,8 @@ SP_v1.4-P1+3_li_2024: SP_v1.4-P1+3+4: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 + ls: dashed colour: brown marker: d @@ -556,7 +570,8 @@ SP_v1.4-P1+3+4: SP_v1.4-P1+3+4_wcs: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 + ls: dashed colour: black marker: d @@ -598,7 +613,8 @@ SP_v1.4-P1+3+4_wcs: SP_v1.4-P1+3+4_no_alpha: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 + ls: dashed colour: black marker: d @@ -640,7 +656,8 @@ SP_v1.4-P1+3+4_no_alpha: SP_v1.4.2: pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 + subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 + ls: dashed colour: black marker: d @@ -682,7 +699,8 @@ SP_v1.4.2: SP_v1.4.1_noleakage: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: dashed colour: black marker: d @@ -702,13 +720,13 @@ SP_v1.4.1_noleakage: w_col: w_iv R: 1.0 star: - path: unions_shapepipe_star_2024_v1.4.a.fits + path: unions_shapepipe_star_2024_v1.4.1.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.1.fits hdu: 1 ra_col: RA dec_col: Dec @@ -724,7 +742,8 @@ SP_v1.4.1_noleakage: SP_v1.4.5: pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 + subdir: /n17data/UNIONS/WL/v1.4.x + ls: dashed colour: dodgerblue marker: d @@ -735,15 +754,17 @@ SP_v1.4.5: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.4370966656902571 shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5.fits + path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 + e1_col_corrected: e1_leak_corrected + e2_col_corrected: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA @@ -767,7 +788,8 @@ SP_v1.4.5: SP_v1.4.5_glass_mock: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/UNIONS/WL/v1.4.x + ls: dashed colour: crimson marker: d @@ -778,7 +800,7 @@ SP_v1.4.5_glass_mock: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.4370966656902571 shear: - path: unions_v1.4.5_glass_mock.fits + path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 @@ -786,50 +808,7 @@ SP_v1.4.5_glass_mock: e2_PSF_col: e2_PSF w_col: w R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.1.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5_leak_corr: - pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 - ls: dashed - colour: teal - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA @@ -853,7 +832,8 @@ SP_v1.4.5_leak_corr: SP_v1.4.5_bright: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/murray/unions_cats + ls: dashed colour: mediumblue marker: d @@ -864,23 +844,23 @@ SP_v1.4.5_bright: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_des + w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -896,7 +876,8 @@ SP_v1.4.5_bright: SP_v1.4.5_faint: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/murray/unions_cats + ls: dashed colour: forestgreen marker: d @@ -907,23 +888,23 @@ SP_v1.4.5_faint: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_des + w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -939,7 +920,8 @@ SP_v1.4.5_faint: SP_v1.4.5_intermediate: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/murray/unions_cats + ls: dashed colour: orange marker: d @@ -950,23 +932,23 @@ SP_v1.4.5_intermediate: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_des + w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -982,7 +964,8 @@ SP_v1.4.5_intermediate: SP_v1.4.5.A: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 + ls: dashed colour: red marker: d @@ -993,23 +976,23 @@ SP_v1.4.5.A: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: ../../../../../../guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5/shapepipe_SPv1.fits + path: shapepipe_SPv1.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: g1 e2_col: g2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w + w_col: w R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1025,7 +1008,8 @@ SP_v1.4.5.A: SP_v1.4.6: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/UNIONS/WL/v1.4.x + ls: dashed colour: green marker: d @@ -1036,15 +1020,17 @@ SP_v1.4.6: n_psf: null # TBD sigma_e: null # TBD shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 + e1_col_corrected: e1_leak_corrected + e2_col_corrected: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA @@ -1052,7 +1038,7 @@ SP_v1.4.6: e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1066,52 +1052,10 @@ SP_v1.4.6: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.4.6_leak_corr: +SP_v1.4.7: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: green - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - shouldn't change with size cuts - n_e: null # TBD - n_psf: null # TBD - sigma_e: null # TBD - shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/UNIONS/WL/v1.4.x -SP_v1.4.7_leak_corr: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ ls: dashed colour: darkorchid marker: d @@ -1122,23 +1066,25 @@ SP_v1.4.7_leak_corr: n_psf: null # TBD sigma_e: null # TBD shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits + path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected + e1_col: e1 + e2_col: e2 + e1_col_corrected: e1_leak_corrected + e2_col_corrected: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1152,9 +1098,10 @@ SP_v1.4.7_leak_corr: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.4.8_leak_corr: +SP_v1.4.8: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/UNIONS/WL/v1.4.x + ls: dashed colour: darkorchid marker: d @@ -1165,23 +1112,25 @@ SP_v1.4.8_leak_corr: n_psf: null # TBD sigma_e: null # TBD shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits + path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected + e1_col: e1 + e2_col: e2 + e1_col_corrected: e1_leak_corrected + e2_col_corrected: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1197,7 +1146,8 @@ SP_v1.4.8_leak_corr: SP_v1.5.4: pipeline: SP - subdir: ../../../UNIONS/WL/v1.5.x + subdir: /n17data/UNIONS/WL/v1.5.x + ls: dashed colour: green marker: d @@ -1237,51 +1187,10 @@ SP_v1.5.4: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.5.4_leak_corr: - pipeline: SP - subdir: ../../../UNIONS/WL/v1.5.x - ls: dashed - colour: darkgreen - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.5.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.5.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - LF_v1.0: pipeline: LF - subdir: CFIS/v1.0/Lensfit + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/Lensfit + ls: solid colour: r marker: "8" @@ -1301,7 +1210,8 @@ LF_v1.0: LF_v2.0: pipeline: LF - subdir: CFIS/v2.0/Lensfit + subdir: /n17data/mkilbing/astro/data/CFIS/v2.0/Lensfit + ls: dashed colour: r marker: p @@ -1319,7 +1229,8 @@ LF_v2.0: SP_matched_LF_v1.0: pipeline: SP - subdir: CFIS/v1.0/matched_LF_SP + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP + ls: dotted colour: magenta marker: v @@ -1352,7 +1263,8 @@ SP_matched_LF_v1.0: LF_matched_SP_v1.0: pipeline: LF - subdir: CFIS/v1.0/matched_LF_SP + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP + ls: dashdot colour: magenta marker: ^ @@ -1372,7 +1284,8 @@ LF_matched_SP_v1.0: SP_v1.0_LFmask_4k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dashdot colour: blue marker: h @@ -1404,7 +1317,8 @@ SP_v1.0_LFmask_4k: SP_v1.0_LFmask_8k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: blue marker: h @@ -1438,7 +1352,8 @@ SP_v1.0_LFmask_8k: SP_v1.3_LFmask_4k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dashdot colour: green marker: h @@ -1470,7 +1385,8 @@ SP_v1.3_LFmask_4k: SP_v1.4-P3_LFmask: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask/P3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask/P3 + ls: dashdot colour: black marker: h @@ -1513,7 +1429,8 @@ SP_v1.4-P3_LFmask: SP_v1.3_LFmask_8k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: green marker: h @@ -1561,7 +1478,8 @@ SP_v1.3_LFmask_8k: SP_v1.3_LFmask_8k_no_alpha: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: blueviolet marker: h @@ -1609,7 +1527,8 @@ SP_v1.3_LFmask_8k_no_alpha: SP_v1.3_LFmask_8k_li_2024: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: deepskyblue marker: h @@ -1657,7 +1576,8 @@ SP_v1.3_LFmask_8k_li_2024: SP_v1.3_LFmask_8k_SN8: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: darkgreen marker: h @@ -1704,7 +1624,8 @@ SP_v1.3_LFmask_8k_SN8: SP_v1.3_LFmask_8k_SN7: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: purple marker: h @@ -1751,7 +1672,8 @@ SP_v1.3_LFmask_8k_SN7: SP_v1.3_LFmask_8k_F2: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: lawngreen marker: h @@ -1798,7 +1720,8 @@ SP_v1.3_LFmask_8k_F2: SP_v1.4_LFmask_8k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: solid colour: black marker: d @@ -1845,7 +1768,8 @@ SP_v1.4_LFmask_8k: SP_v1.4_LFmask_8k_noalpha: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: solid colour: brown marker: d @@ -1892,7 +1816,8 @@ SP_v1.4_LFmask_8k_noalpha: SP_matched_MP_v1.0: pipeline: SP - subdir: CFIS/v1.0/matched_SP_MP + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_SP_MP + ls: dashdot colour: lightblue marker: D @@ -1918,7 +1843,8 @@ SP_matched_LF_fp_v1.0: SP_axel_v0.0: pipeline: SP - subdir: CFIS/v0.0 + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + ls: dotted colour: cyan marker: > @@ -1963,7 +1889,8 @@ SP_axel_v0.0: SP_v0.1.1: pipeline: SP - subdir: CFIS/v0.0 + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + ls: dashdot colour: cyan marker: v @@ -1997,7 +1924,8 @@ SP_v0.1.1: DES: pipeline: SP - subdir: DES + subdir: /n17data/mkilbing/astro/data/DES + ls: solid colour: orange marker: "*" diff --git a/notebooks/cosmo_val/run_cosmo_val.py b/notebooks/cosmo_val/run_cosmo_val.py index 3e2aa63..3513e22 100644 --- a/notebooks/cosmo_val/run_cosmo_val.py +++ b/notebooks/cosmo_val/run_cosmo_val.py @@ -19,11 +19,11 @@ # %% cv = CosmologyValidation( - versions=["SP_v1.4.5"], + versions=["SP_v1.4.5_leak_corr", "SP_v1.4.6_leak_corr", "SP_v1.4.7_leak_corr"], data_base_dir="/n17data/mkilbing/astro/data", - npatch=1, - theta_min=1, - theta_max=250, + npatch=256, + theta_min=1.0, + theta_max=250.0, nbins=20, theta_min_plot=1, theta_max_plot=250, diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index c57b833..78a9d89 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,5 +1,7 @@ # %% +import copy import os +from pathlib import Path import colorama import healpy as hp @@ -25,7 +27,6 @@ class CosmologyValidation: def __init__( self, versions, - data_base_dir, catalog_config="./cat_config.yaml", output_dir=None, rho_tau_method="lsq", @@ -49,10 +50,8 @@ def __init__( pol_factor=True, nrandom_cell=10, cosmo_params=None, - redshift_file=None + redshift_file=None, ): - self.versions = versions - self.data_base_dir = data_base_dir self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method self.compute_cov_rho = compute_cov_rho @@ -99,15 +98,54 @@ def __init__( with open(catalog_config, "r") as file: self.cc = cc = yaml.load(file.read(), Loader=yaml.FullLoader) - for ver in ["nz", *versions]: - if ver not in cc: - raise KeyError( - f"Version string {ver} not found in config file{catalog_config}" - ) - version_base = f"{data_base_dir}/{cc[ver]['subdir']}" + def resolve_paths_for_version(ver): + """Resolve relative paths for a version using its subdir.""" + subdir = Path(cc[ver]["subdir"]) for key in cc[ver]: if "path" in cc[ver][key]: - cc[ver][key]["path"] = f"{version_base}/{cc[ver][key]['path']}" + path = Path(cc[ver][key]["path"]) + cc[ver][key]["path"] = ( + str(path) if path.is_absolute() else str(subdir / path) + ) + + resolve_paths_for_version("nz") + processed = {"nz"} + final_versions = [] + + for ver in versions: + if ver.endswith("_leak_corr"): + base_ver = ver.replace("_leak_corr", "") + target = base_ver + if base_ver not in cc: + raise KeyError( + f"Base version {base_ver} not found for {ver} in config file " + f"{catalog_config}" + ) + if "e1_col_corrected" not in cc[base_ver]["shear"]: + raise ValueError( + f"{base_ver} does not have e1_col_corrected/e2_col_corrected " + f"fields; cannot create {ver}" + ) + else: + target = ver + if ver not in cc: + raise KeyError( + f"Version string {ver} not found in config file " + f"{catalog_config}" + ) + + if target not in processed: + resolve_paths_for_version(target) + processed.add(target) + + if ver.endswith("_leak_corr"): + cc[ver] = copy.deepcopy(cc[base_ver]) + cc[ver]["shear"]["e1_col"] = cc[base_ver]["shear"]["e1_col_corrected"] + cc[ver]["shear"]["e2_col"] = cc[base_ver]["shear"]["e2_col_corrected"] + + final_versions.append(ver) + + self.versions = final_versions # Override output directory if provided if output_dir is not None: @@ -1927,7 +1965,6 @@ def plot_cosebis( plot_cosebis_covariance_matrix, plot_cosebis_modes, plot_cosebis_scale_cut_heatmap, - scale_cut_to_bins, ) # Use instance defaults if not specified @@ -1937,8 +1974,8 @@ def plot_cosebis( # Determine variance method based on whether theoretical covariance is used var_method = "analytic" if cov_path is not None else "jackknife" - - # Create output filename with integration parameters to match Snakemake expectations + + # Create output filename with integration parameters to match Snakemake out_stub = ( f"{output_dir}/{version}_cosebis_minsep={min_sep_int}_" f"maxsep={max_sep_int}_nbins={nbins_int}_npatch={npatch}_" @@ -1976,7 +2013,9 @@ def plot_cosebis( # Multiple scale cuts: use fiducial_scale_cut if provided, otherwise use # full range if fiducial_scale_cut is not None: - plot_results = results[find_conservative_scale_cut_key(results, fiducial_scale_cut)] + plot_results = results[ + find_conservative_scale_cut_key(results, fiducial_scale_cut) + ] else: # Use full range result (largest scale cut) max_range_key = max(results.keys(), key=lambda x: x[1] - x[0]) @@ -2010,7 +2049,9 @@ def plot_cosebis( "max_sep": max_sep or self.treecorr_config["max_sep"], "nbins": nbins or self.treecorr_config["nbins"], } - gg_temp = self.calculate_2pcf(version, npatch=npatch, **treecorr_config_temp) + gg_temp = self.calculate_2pcf( + version, npatch=npatch, **treecorr_config_temp + ) plot_cosebis_scale_cut_heatmap( results, @@ -2053,9 +2094,10 @@ def calculate_pseudo_cl_eb_cov(self): self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") lmax = 2 * self.nside - path_redshift_distr = ( - self.data_base_dir + self.cc[ver]["shear"]["redshift_distr"] - ) + redshift_path = Path(self.cc[ver]["shear"]["redshift_distr"]) + if not redshift_path.is_absolute(): + redshift_path = Path(self.cc[ver]["subdir"]) / redshift_path + path_redshift_distr = str(redshift_path) pw = hp.pixwin(nside, lmax=lmax) # Load redshift distribution and calculate theory C_ell diff --git a/src/sp_validation/tests/test_catalog_paths.py b/src/sp_validation/tests/test_catalog_paths.py new file mode 100644 index 0000000..a66778c --- /dev/null +++ b/src/sp_validation/tests/test_catalog_paths.py @@ -0,0 +1,97 @@ +"""Lightweight checks that catalog paths referenced in ``cat_config.yaml`` exist. + +These tests do not load the data; they only ensure that the configuration points +to readable files so that downstream validation runs fail fast if a path drifts. +""" + +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path +from typing import Dict, Iterator, Tuple + +import yaml + + +CATALOG_CONFIG = ( + Path(__file__).resolve().parents[3] / "notebooks" / "cosmo_val" / "cat_config.yaml" +) + +EXPECTED_MISSING = { + "SP_v1.0": {"shear", "psf"}, + "SP_v1.1": {"shear", "psf"}, + "SP_v1.4": {"shear", "star", "psf"}, + "SP_v1.4_conv": {"shear", "star", "psf"}, + "SP_v1.4_noalpha": {"shear", "star", "psf"}, + "SP_v1.4-P1+3": {"shear", "star", "psf"}, + "SP_v1.4-P1+3_wcs": {"shear", "star", "psf"}, + "SP_v1.4-P1+3_no_alpha": {"shear", "star", "psf"}, + "SP_v1.4-P1+3_li_2024": {"shear", "star", "psf"}, + "SP_v1.4-P1+3+4": {"shear", "star", "psf"}, + "SP_v1.4-P1+3+4_wcs": {"shear", "star", "psf"}, + "SP_v1.4-P1+3+4_no_alpha": {"shear", "star", "psf"}, + "SP_matched_MP_v1.0": {"shear"}, +} + + +def _resolve(base: Path, candidate: str) -> Path: + """Return an absolute path given a base directory and a candidate string.""" + candidate_path = Path(candidate) + return candidate_path if candidate_path.is_absolute() else base / candidate_path + + +def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: + """Yield (name, entry) pairs for catalog-like entries in the config.""" + for name, entry in config.items(): + if not isinstance(entry, dict): + continue + if "subdir" not in entry: + continue + yield name, entry + + +def test_catalog_files_exist(): + """Ensure every shear/star/psf entry references an on-disk file.""" + config = yaml.safe_load(CATALOG_CONFIG.read_text()) + + missing = defaultdict(set) + + for name, entry in _iter_catalog_entries(config): + base = Path(entry["subdir"]) + assert base.is_absolute(), f"{name}: subdir must be absolute ({base})" + + if name == "nz": + dndz_path = _resolve(base, entry["dndz"]["path"]) + assert ( + dndz_path.parent.is_dir() + ), f"{name}: dndz parent directory missing ({dndz_path.parent})" + continue + + for block_name in ("shear", "star", "psf"): + block = entry.get(block_name) + if not block: + continue + resolved_path = _resolve(base, block["path"]) + if not resolved_path.is_file(): + missing[name].add(block_name) + + unexpected = { + name: blocks - EXPECTED_MISSING.get(name, set()) + for name, blocks in missing.items() + if blocks - EXPECTED_MISSING.get(name, set()) + } + assert not unexpected, ( + "Unexpected missing catalog files detected: " + f"{ {name: sorted(blocks) for name, blocks in unexpected.items()} }" + ) + + resolved = { + name: expected - missing.get(name, set()) + for name, expected in EXPECTED_MISSING.items() + if expected - missing.get(name, set()) + } + assert not resolved, ( + "Previously missing catalog files are now available; " + f"update EXPECTED_MISSING to reflect this change: " + f"{ {name: sorted(blocks) for name, blocks in resolved.items()} }" + ) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py new file mode 100644 index 0000000..55b8f80 --- /dev/null +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -0,0 +1,127 @@ +"""UNIT TESTS FOR COSMOLOGY VALIDATION CLASS. + +This module contains integration tests for the CosmologyValidation class, +specifically testing the ellipticity_suffix parameter functionality for +handling leak-corrected ellipticity columns. + +:Author: cdaley + +""" + +import os + +import pytest + +from sp_validation.cosmo_val import CosmologyValidation + + +class TestCosmologyValidation: + """Test CosmologyValidation initialization and additive bias calculation.""" + + @pytest.fixture + def base_config(self, tmp_path): + """Common configuration parameters for tests.""" + # Get the path to the repo root (3 levels up from this test file) + repo_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + ) + catalog_config = os.path.join( + repo_root, "notebooks", "cosmo_val", "cat_config.yaml" + ) + + # Use temporary directory for outputs + output_dir = tmp_path / "test_output" + output_dir.mkdir() + + return { + "catalog_config": catalog_config, + "output_dir": str(output_dir), + "npatch": 1, + "theta_min": 1.0, + "theta_max": 250.0, + "nbins": 20, + } + + @pytest.mark.parametrize( + "version,e1_col,e2_col", + [ + ("SP_v1.4.5", "e1", "e2"), + ("SP_v1.4.6", "e1", "e2"), + ("SP_v1.4.5_glass_mock", "e1", "e2"), + ("SP_v1.4.5_bright", "e1", "e2"), + ("SP_v1.4.5_faint", "e1", "e2"), + ("SP_v1.4.5_intermediate", "e1", "e2"), + ("SP_v1.4.5.A", "g1", "g2"), + ("SP_v1.4.7", "e1", "e2"), + ("SP_v1.4.8", "e1", "e2"), + ], + ) + def test_additive_bias_base_columns(self, base_config, version, e1_col, e2_col): + """Test additive bias calculation using base ellipticity columns. + + This test initializes CosmologyValidation without an ellipticity_suffix, + which means it will use the default columns defined in the catalog + configuration. + """ + cv = CosmologyValidation( + versions=[version], + **base_config, + ) + + # Verify version names remain unchanged + assert cv.versions == [version] + + # Verify the ellipticity columns are the base columns + assert cv.cc[version]["shear"]["e1_col"] == e1_col + assert cv.cc[version]["shear"]["e2_col"] == e2_col + + # Calculate additive bias + cv.calculate_additive_bias() + + # Verify c1 and c2 were calculated and stored + assert hasattr(cv, "_c1") and hasattr(cv, "_c2") + assert version in cv.c1 + assert version in cv.c2 + + # Verify the values are numeric (not NaN or None) + assert isinstance(cv.c1[version], float) + assert isinstance(cv.c2[version], float) + + @pytest.mark.parametrize("version", ["SP_v1.4.5"]) + def test_additive_bias_leak_corrected_columns(self, base_config, version): + """Test additive bias calculation using leak-corrected columns. + + This test requests a leak-corrected version by passing "{version}_leak_corr" + as the version name. The CosmologyValidation class automatically detects + the _leak_corr suffix and creates the config entry using e1_col_corrected + and e2_col_corrected from the base version. + """ + version_leak_corr = f"{version}_leak_corr" + + cv = CosmologyValidation( + versions=[version_leak_corr], + **base_config, + ) + + # Verify version names include the _leak_corr suffix + assert cv.versions == [version_leak_corr] + + # Verify the leak-corrected config was auto-created with corrected columns + assert cv.cc[version_leak_corr]["shear"]["e1_col"] == "e1_leak_corrected" + assert cv.cc[version_leak_corr]["shear"]["e2_col"] == "e2_leak_corrected" + + # Verify original config entry remains unchanged + assert cv.cc[version]["shear"]["e1_col"] == "e1" + assert cv.cc[version]["shear"]["e2_col"] == "e2" + + # Calculate additive bias + cv.calculate_additive_bias() + + # Verify c1 and c2 were calculated and stored with leak-corrected name + assert hasattr(cv, "_c1") and hasattr(cv, "_c2") + assert version_leak_corr in cv.c1 + assert version_leak_corr in cv.c2 + + # Verify the values are numeric + assert isinstance(cv.c1[version_leak_corr], float) + assert isinstance(cv.c2[version_leak_corr], float) From 62868c0777a894dc7e4d1f25b737249c3f62ec73 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Oct 2025 00:17:34 +0200 Subject: [PATCH 04/34] gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index df94531..09e4ece 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,6 @@ notebooks/demo_check_footprint.ipynb notebooks/demo_calibrate_minimal_cat.ipynb notebooks/leakage_minimal.ipynb notebooks/demo_add_bands.ipynb + +CLAUDE.md +AGENTS.md \ No newline at end of file From 0ce82ef44853ca7ca231c3e01882a6563d5f5893 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 13 Oct 2025 18:57:16 +0200 Subject: [PATCH 05/34] remove PR draft file --- docs/PR_DRAFT.md | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 docs/PR_DRAFT.md diff --git a/docs/PR_DRAFT.md b/docs/PR_DRAFT.md deleted file mode 100644 index 38b1bfe..0000000 --- a/docs/PR_DRAFT.md +++ /dev/null @@ -1,38 +0,0 @@ -# Draft PR: Catalog Path Cleanup & Guardrails - -## Summary - -- Normalised every `subdir` entry in `notebooks/cosmo_val/cat_config.yaml` so the loader relies on explicit `/n17data/...` locations rather than the old `data_dir` fallback. Shared resources (redshift distributions, common star/psf FITS) now point at their canonical mounts. -- Updated the Pseudo-`C_\ell` code path in `src/sp_validation/cosmo_val.py` to read redshift files directly from the catalog configuration, removing the `data_base_dir` assumption. -- Added `src/sp_validation/tests/test_catalog_paths.py`, a lightweight validator that resolves each shear/star/psf reference and records a small allow-list for catalogues whose source files are currently unavailable. -- Kept the additive-bias regression test focused on the catalogues that still have data on disk; the missing releases stay excluded so CI remains green. - -## Outstanding Data Gaps - -All of the catalogues below are missing their **shear** FITS on the shared mounts (star/psf often missing as well): - -1. `SP_v1.0`, `SP_v1.1`, and `SP_matched_MP_v1.0` -2. The `SP_v1.4` family: `SP_v1.4`, `SP_v1.4_conv`, `SP_v1.4_noalpha` -3. Every `SP_v1.4-P1+3*` derivative (same missing 2022-era inputs as item 2) - -These entries remain in `cat_config.yaml` for provenance, are listed in the validator allow-list, and are intentionally omitted from the additive-bias parameter set. The follow-up PR should either restore their data to `/n17data/...`, retarget them at new releases (e.g. the 2024 v1.4.1 products), or mark them as deprecated. - -## Proposed Next Steps - -1. **Decide The Fate Of The Legacy Catalogues** - - Recover the missing FITS/parquet files, or formally deprecate the versions above (update docs/tests accordingly). - -2. **Re-home The v1.4 Variants (if data exist)** - - Point each affected block at the correct `/n17data/UNIONS/WL/v1.4.x//` directory. - - Prefer catalogue-local star/psf fits when the directory is more than a symlink; fall back to the shared root otherwise. - - Remove the entries from `EXPECTED_MISSING` and re-enable them in regression tests once their files resolve. - -3. **Document The Conventions** - - Add a short developer note describing the absolute-path policy, preferred star/psf strategy, and how to run the new path validator before pushing catalog edits. - - Optionally list the currently offline datasets so collaborators know what needs recovering. - -## Validation Checklist - -- `app python -m pytest src/sp_validation/tests/test_catalog_paths.py` -- `app python -m pytest src/sp_validation/tests/test_cosmo_val.py -k additive_bias` -- Manual `ls` spot-check of any retargeted catalogue directories to ensure we pick up the intended local star/psf files rather than parent symlinks. From 6a44359650d9a66b1426ef6c2c844d8f12b319c1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 13 Oct 2025 19:05:27 +0200 Subject: [PATCH 06/34] propagate changes to run_cosmo_val --- notebooks/cosmo_val/run_cosmo_val.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/notebooks/cosmo_val/run_cosmo_val.py b/notebooks/cosmo_val/run_cosmo_val.py index 3513e22..037a38d 100644 --- a/notebooks/cosmo_val/run_cosmo_val.py +++ b/notebooks/cosmo_val/run_cosmo_val.py @@ -20,8 +20,7 @@ # %% cv = CosmologyValidation( versions=["SP_v1.4.5_leak_corr", "SP_v1.4.6_leak_corr", "SP_v1.4.7_leak_corr"], - data_base_dir="/n17data/mkilbing/astro/data", - npatch=256, + npatch=1, theta_min=1.0, theta_max=250.0, nbins=20, From 746479b54264677a38127cd4b7af5fa69c8471d9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 15 Oct 2025 00:11:54 +0200 Subject: [PATCH 07/34] Prune legacy catalogs and tighten survey stats --- notebooks/cosmo_val/cat_config.yaml | 3276 +++++++---------- notebooks/cosmo_val/compute_theory_cov.py | 4 +- src/sp_validation/cosmo_val.py | 132 +- src/sp_validation/tests/test_catalog_paths.py | 38 +- 4 files changed, 1487 insertions(+), 1963 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 493b5db..dd02dfb 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1,1971 +1,1397 @@ -paths: - output: ./output +DES: + subdir: /n17data/mkilbing/astro/data/DES + pipeline: SP + colour: orange + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: '*' + cov_th: + A: 4143 + n_e: 5.67 + n_psf: 3.8 + sigma_e: 0.29 + path_rho: rho_stats_DES.fits + path_tau: tau_stats_DES.fits + path_xi_plus: xi_plus_DES.fits + path_xi_minus: xi_minus_DES.fits + psf: + PSF_size: piff_T + square_size: false + star_size: obs_T + hdu: 1 + path: psf_y3a1-v29.fits + ra_col: ra + dec_col: dec + e1_PSF_col: piff_e1 + e1_star_col: obs_e1 + e2_PSF_col: piff_e2 + e2_star_col: obs_e2 + shear: + R: 1.0 + R11: R11 + R22: R22 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: DES_Y3_cut.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_psf + e2_col: e2 + e2_PSF_col: e2_psf + label: DES-Y3 + star: + ra_col: ra + dec_col: dec + e1_col: obs_e1 + e2_col: obs_e2 + path: psf_y3a1-v29.fits -nz: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz +LF_matched_SP_v1.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP + pipeline: LF + colour: magenta + getdist_colour: 0.0, 0.5, 0.0 + ls: dashdot + marker: ^ + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt + path: masked_lensfit_goldshape_2022v1.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: full_starcat_tmp_1_10_seed_1234.fits - dndz: - path: dndz - blind: A +LF_v1.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/Lensfit + pipeline: LF + colour: r + getdist_colour: 1.0, 0.5, 0.0 + ls: solid + marker: '8' + shear: + R: 1.0 + covmat_file: ./covs/lensfit_A/cov_lensfit_A.txt + path: lensfit_goldshape_2022v1.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: full_starcat_tmp_1_10_seed_1234.fits -SP_test: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +LF_v2.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v2.0/Lensfit + pipeline: LF + colour: r + ls: dashed + marker: p + shear: + R: 1.0 + path: lensfit_goldshape_2022v2_psf1.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1_psf + e2_col: e2_psf + path: Pall_hdu1_THELI_psfs1_mosaic.fits - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.1.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_axel_v0.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dotted + marker: '' + cov_th: + A: 1500 + n_e: 6.47 + n_psf: 0.47 + sigma_e: 0.3 + path_rho: rho_stats_SP_v0.0.fits + path_tau: tau_stats_SP_v0.0.fits + path_xi_plus: xi_plus_SP_v0.0.fits + path_xi_minus: xi_minus_SP_v0.0.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: star_cat.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: shapepipe_1500_goldshape_v1.fits + w_col: w + e1_col: g1 + e1_PSF_col: E1_PSF_HSM + e2_col: g2 + e2_PSF_col: E2_PSF_HSM + star: + ra_col: RA + dec_col: DEC + e1_col: E1_STAR_HSM + e2_col: E2_STAR_HSM + path: star_cat.fits -SP_v1.0: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_matched_LF_v1.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP + pipeline: SP + colour: magenta + getdist_colour: 0.90340530805574, 0.06198853997875753, 0.6527408678603205 + ls: dotted + marker: v + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt + path: masked_matched_unions_shapepipe_extended_2022_v1.0.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ls: solid - colour: blue - marker: o - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_2022_v1.0.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" +SP_test: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.1.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.1.fits -SP_v1.1: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v0.1.1: + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: v + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: star_cat.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_extended_2022_v0.0_recon_theli_8192.fits + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: DEC + e1_col: E1_STAR_HSM + e2_col: E2_STAR_HSM + path: star_cat.fits - ls: dashed - colour: b - marker: s - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_2022_v1.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" +SP_v1.0_LFmask_4k: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: blue + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dashdot + marker: h + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_2022_v1.0_mtheli4k.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits -SP_v1.3: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v1.0_LFmask_8k: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: blue + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_extended_2022_v1.0_mtheli8k.fits + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ls: solid - colour: green - marker: p - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_extended_2022_v1.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.3: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: p + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_v1.3.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3.fits -SP_v1.4: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v1.3_LFmask_4k: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: green + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dashdot + marker: h + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_2022_v1.3_mtheli4k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.3_LFmask_8k: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: green + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 7.6 + n_psf: 0.68 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_2022_v1.3_mtheli8k.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits -SP_v1.4_conv: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v1.3_LFmask_8k_F2: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: lawngreen + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 9.4 + n_psf: 0.48 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_2022_f2_v1.3_mtheli8k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits + label: SP_LFmask_star - ls: solid - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.3_LFmask_8k_SN7: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: purple + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 10.989 + n_psf: 0.48 + sigma_e: 0.3796 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_2022_sn7_v1.3_mtheli8k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits + label: SP_LFmask_star -SP_v1.4_noalpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v1.3_LFmask_8k_SN8: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: darkgreen + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 9.7 + n_psf: 0.48 + sigma_e: 0.35 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_2022_sn8_v1.3_mtheli8k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits + label: SP_LFmask_star - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.3_LFmask_8k_li_2024: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: deepskyblue + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 7.6 + n_psf: 0.68 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_rmalpha_li_2022_v1.3_mtheli8k.fits + w_col: w + e1_col: e1_cor_2 + e1_PSF_col: e1_PSF + e2_col: e2_cor_2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits +SP_v1.3_LFmask_8k_no_alpha: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: blueviolet + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 7.6 + n_psf: 0.68 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_rmalpha_2022_v1.3_mtheli8k.fits + w_col: w + e1_col: e1_cor + e1_PSF_col: e1_PSF + e2_col: e2_cor + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits SP_v1.4-P3: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 - - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 337 #deg^2 - n_e: 7.45 #arcmin-2 - n_psf: 0.34 #arcmin-2 - sigma_e: 0.32 - path_rho: rho_stats_SP_v1.4-P3.fits - path_tau: tau_stats_SP_v1.4-P3.fits - path_xi_plus: xi_plus_SP_v1.4-P3.fits - path_xi_minus: xi_minus_SP_v1.4-P3.fits - shear: - path: unions_shapepipe_extended_2022_P3_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 - - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.4-P1+3.fits - path_tau: tau_stats_SP_v1.4-P1+3.fits - path_xi_plus: xi_plus_SP_v1.4-P1+3.fits - path_xi_minus: xi_minus_SP_v1.4-P1+3.fits - shear: - path: unions_shapepipe_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P1+3_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_wcs: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 - - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_no_alpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 + pipeline: SP + colour: brown + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 337 + n_e: 7.45 + n_psf: 0.34 + sigma_e: 0.32 + path_rho: rho_stats_SP_v1.4-P3.fits + path_tau: tau_stats_SP_v1.4-P3.fits + path_xi_plus: xi_plus_SP_v1.4-P3.fits + path_xi_minus: xi_minus_SP_v1.4-P3.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 2 + path: unions_shapepipe_psf_2022_P3_v1.4.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_P3_v1.4.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_P3_v1.4.fits - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_rmalpha_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_li_2024: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 - - ls: dashed - colour: gold - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_rmalpha_li_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 - - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4_wcs: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 - - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_extended_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4_no_alpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 - - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_extended_rmalpha_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.2: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 - - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.2.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.4-P3_LFmask: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask/P3 + pipeline: SP + colour: black + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dashdot + marker: h + cov_th: + A: 223 + n_e: 7.44 + n_psf: 0.32 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.4-P3_LFmask.fits + path_tau: tau_stats_SP_v1.4-P3_LFmask.fits + path_xi_plus: xi_plus_SP_v1.4-P3_LFmask.fits + path_xi_minus: xi_minus_SP_v1.4-P3_LFmask.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.4_mtheli4k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_2022_v1.4_mtheli4k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.4_mtheli4k.fits SP_v1.4.1_noleakage: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 3191.1827474082734 + n_e: 7.0582373812033286 + n_psf: 0.5705736293858423 + sigma_e: 0.317469332009887 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.1.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_2024_v1.4.1.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.1.fits - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3191.1827474082734 #deg^2 - n_e: 7.0582373812033286 #arcmin-2 - n_psf: 0.5705736293858423 #arcmin-2 - sigma_e: 0.317469332009887 - shear: - path: unions_shapepipe_2024_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.4.1.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.4.2: + subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 1826 + n_e: 7.75 + n_psf: 0.35 + sigma_e: 0.3133 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.2.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: dodgerblue - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_col_corrected: e1_leak_corrected - e2_col_corrected: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: dodgerblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2582.310451583689 + n_e: 7.748708180391998 + n_psf: 0.4973044366748657 + sigma_e: 0.4375145387558129 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.5_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits -SP_v1.4.5_glass_mock: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: crimson - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.4.5.A: + subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 + pipeline: SP + colour: red + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: shapepipe_SPv1.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: g1 + e1_PSF_col: e1_PSF + e2_col: g2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_bright: - pipeline: SP - subdir: /n17data/murray/unions_cats - - ls: dashed - colour: mediumblue - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: mediumblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_faint: - pipeline: SP - subdir: /n17data/murray/unions_cats + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: forestgreen + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ls: dashed - colour: forestgreen - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.4.5_glass_mock: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: crimson + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2782 + n_e: 7.18 + n_psf: 0.752316232272063 + sigma_e: 0.4370966656902571 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_intermediate: - pipeline: SP - subdir: /n17data/murray/unions_cats - - ls: dashed - colour: orange - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5.A: - pipeline: SP - subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 - - ls: dashed - colour: red - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: shapepipe_SPv1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: g1 - e2_col: g2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: orange + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.6: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: green - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - shouldn't change with size cuts - n_e: null # TBD - n_psf: null # TBD - sigma_e: null # TBD - shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_col_corrected: e1_leak_corrected - e2_col_corrected: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.7: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: darkorchid - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: null # TBD - n_e: null # TBD - n_psf: null # TBD - sigma_e: null # TBD - shear: - path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_col_corrected: e1_leak_corrected - e2_col_corrected: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkorchid + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2050.5970269635545 + n_e: 6.121543643587931 + n_psf: 0.752316232272063 + sigma_e: 0.3797419081703805 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.7_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.8: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: darkorchid - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: null # TBD - n_e: null # TBD - n_psf: null # TBD - sigma_e: null # TBD - shear: - path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_col_corrected: e1_leak_corrected - e2_col_corrected: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.5.4: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.5.x - - ls: dashed - colour: green - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.5.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.5.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -LF_v1.0: - pipeline: LF - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/Lensfit - - ls: solid - colour: r - marker: "8" - getdist_colour: 1.0, 0.5, 0.0 - shear: - path: lensfit_goldshape_2022v1.fits - covmat_file: ./covs/lensfit_A/cov_lensfit_A.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: full_starcat_tmp_1_10_seed_1234.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - -LF_v2.0: - pipeline: LF - subdir: /n17data/mkilbing/astro/data/CFIS/v2.0/Lensfit - - ls: dashed - colour: r - marker: p - shear: - path: lensfit_goldshape_2022v2_psf1.fits - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: Pall_hdu1_THELI_psfs1_mosaic.fits - ra_col: RA - dec_col: Dec - e1_col: e1_psf - e2_col: e2_psf - -SP_matched_LF_v1.0: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP - - ls: dotted - colour: magenta - marker: v - getdist_colour: 0.90340530805574, 0.06198853997875753, 0.6527408678603205 - shear: - path: masked_matched_unions_shapepipe_extended_2022_v1.0.fits - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -LF_matched_SP_v1.0: - pipeline: LF - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP - - ls: dashdot - colour: magenta - marker: ^ - getdist_colour: 0.0, 0.5, 0.0 - shear: - path: masked_lensfit_goldshape_2022v1.fits - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: full_starcat_tmp_1_10_seed_1234.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - -SP_v1.0_LFmask_4k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dashdot - colour: blue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_2022_v1.0_mtheli4k.fits - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.0_LFmask_8k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: blue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_extended_2022_v1.0_mtheli8k.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.3_LFmask_4k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dashdot - colour: green - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_2022_v1.3_mtheli4k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.4-P3_LFmask: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask/P3 - - ls: dashdot - colour: black - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 223 #deg^2 - n_e: 7.44 #arcmin-2 - n_psf: 0.32 #arcmin-2 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.4-P3_LFmask.fits - path_tau: tau_stats_SP_v1.4-P3_LFmask.fits - path_xi_plus: xi_plus_SP_v1.4-P3_LFmask.fits - path_xi_minus: xi_minus_SP_v1.4-P3_LFmask.fits - shear: - path: unions_shapepipe_2022_v1.4_mtheli4k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.4_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.3_LFmask_8k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: green - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_no_alpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: blueviolet - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.3_mtheli8k.fits - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_li_2024: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: deepskyblue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_rmalpha_li_2022_v1.3_mtheli8k.fits - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_SN8: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: darkgreen - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 9.7 - n_psf: 0.48 - sigma_e: 0.35 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_sn8_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_SN7: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: purple - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 10.989 - n_psf: 0.48 - sigma_e: 0.3796 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_sn7_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_F2: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: lawngreen - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 9.4 - n_psf: 0.48 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_f2_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkorchid + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2049.034211379564 + n_e: 6.121484916053109 + n_psf: 0.752316232272063 + sigma_e: 0.37973978258052915 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.8_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4_LFmask_8k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: solid - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2137.7977618140676 #deg^2 - n_e: 7.976462506484096 #arcmin-2 - n_psf: 0.5434016250405327 #arcmin-2 - sigma_e: 0.31509572849714534 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0_mtheli8k.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: d + cov_th: + A: 2137.7977618140676 + n_e: 7.976462506484096 + n_psf: 0.5434016250405327 + sigma_e: 0.31509572849714534 + path_rho: rho_stats_SP_v1.4.fits + path_tau: tau_stats_SP_v1.4.fits + path_xi_plus: xi_plus_SP_v1.4.fits + path_xi_minus: xi_minus_SP_v1.4.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_v1.4.0_mtheli8k.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits SP_v1.4_LFmask_8k_noalpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2137.7977618140676 #deg^2 - n_e: 7.976462506484096 #arcmin-2 - n_psf: 0.5434016250405327 #arcmin-2 - sigma_e: 0.31509572849714534 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.4.0_mtheli8k.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_matched_MP_v1.0: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_SP_MP - - ls: dashdot - colour: lightblue - marker: D - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: cfis-shapepipe.parquet - e1_col: e1 - e2_col: e2 - R: 1.0 - -SP_matched_LF_fp_v1.0: - pipeline: SP - ls: dashed - colour: lightblue - marker: < - getdist_colour: 0.08130806428547999, 0.1463709449188496, 0.674947827197815 - shear: - path: matched_footprint_shapepipe.fits - covmat_file: ./covs/lensfit_matched/cov_lensfit_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - -SP_axel_v0.0: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: brown + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: d + cov_th: + A: 2137.7977618140676 + n_e: 7.976462506484096 + n_psf: 0.5434016250405327 + sigma_e: 0.31509572849714534 + path_rho: rho_stats_SP_v1.4.fits + path_tau: tau_stats_SP_v1.4.fits + path_xi_plus: xi_plus_SP_v1.4.fits + path_xi_minus: xi_minus_SP_v1.4.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_rmalpha_2022_v1.4.0_mtheli8k.fits + w_col: w + e1_col: e1_cor + e1_PSF_col: e1_PSF + e2_col: e2_cor + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - ls: dotted - colour: cyan - marker: > - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1500 - n_e: 6.47 - n_psf: 0.47 - sigma_e: 0.30 - path_rho: rho_stats_SP_v0.0.fits - path_tau: tau_stats_SP_v0.0.fits - path_xi_plus: xi_plus_SP_v0.0.fits - path_xi_minus: xi_minus_SP_v0.0.fits - shear: - path: shapepipe_1500_goldshape_v1.fits - e1_col: g1 - e2_col: g2 - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - w_col: w - R: 1.0 - star: - path: star_cat.fits - ra_col: RA - dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM - psf: - path: star_cat.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - square_size: True - -SP_v0.1.1: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 - - ls: dashdot - colour: cyan - marker: v - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_extended_2022_v0.0_recon_theli_8192.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - star: - path: star_cat.fits - ra_col: RA - dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM - psf: - path: star_cat.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM +SP_v1.5.4: + subdir: /n17data/UNIONS/WL/v1.5.x + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.5.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.5.a.fits -DES: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/DES +nz: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz + dndz: + blind: A + path: dndz - ls: solid - colour: orange - marker: "*" - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 4143 - n_e: 5.67 - n_psf: 3.8 - sigma_e: 0.29 - path_rho: rho_stats_DES.fits - path_tau: tau_stats_DES.fits - path_xi_plus: xi_plus_DES.fits - path_xi_minus: xi_minus_DES.fits - shear: - path: DES_Y3_cut.fits - label: DES-Y3 - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_psf - e2_PSF_col: e2_psf - w_col: w - R: 1.0 - R11: R11 - R22: R22 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - star: - path: psf_y3a1-v29.fits - ra_col: ra - dec_col: dec - e1_col: obs_e1 - e2_col: obs_e2 - psf: - path: psf_y3a1-v29.fits - hdu: 1 - ra_col: ra - dec_col: dec - e1_PSF_col: piff_e1 - e2_PSF_col: piff_e2 - e1_star_col: obs_e1 - e2_star_col: obs_e2 - PSF_size: piff_T - star_size: obs_T - square_size: False +paths: + output: ./output diff --git a/notebooks/cosmo_val/compute_theory_cov.py b/notebooks/cosmo_val/compute_theory_cov.py index 2c93d47..8c20a73 100644 --- a/notebooks/cosmo_val/compute_theory_cov.py +++ b/notebooks/cosmo_val/compute_theory_cov.py @@ -41,7 +41,7 @@ def get_params_rho_tau(cat, survey="other"): base_dir = '/home/guerrini/data/' - versions = ['SP_v1.4-P3', 'SP_v1.4-P3_LFmask', 'SP_v1.4-P1+3', 'SP_v1.3_LFmask_8k', 'SP_axel_v0.0', 'DES'] + versions = ['SP_v1.4-P3', 'SP_v1.4-P3_LFmask', 'SP_v1.3_LFmask_8k', 'SP_axel_v0.0', 'DES'] path_config = '/home/guerrini/sp_validation/notebooks/cosmo_val/cat_config.yaml' output_dir = '/home/guerrini/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/' @@ -94,4 +94,4 @@ def get_params_rho_tau(cat, survey="other"): print("--- Covariance computation %s seconds ---" % (time.time() - start_time)) np.save(output_dir+'/cov_tau_'+ver+'_th.npy', cov) - print("Saved covariance matrix of version: ", ver) \ No newline at end of file + print("Saved covariance matrix of version: ", ver) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 78a9d89..4527532 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -95,7 +95,8 @@ def __init__( "cross_patch_weight": "match" if var_method == "jackknife" else "simple", } - with open(catalog_config, "r") as file: + self.catalog_config_path = Path(catalog_config) + with self.catalog_config_path.open("r") as file: self.cc = cc = yaml.load(file.read(), Loader=yaml.FullLoader) def resolve_paths_for_version(ver): @@ -154,6 +155,135 @@ def resolve_paths_for_version(ver): if not os.path.exists(cc["paths"]["output"]): os.mkdir(cc["paths"]["output"]) + def compute_survey_stats( + self, + ver, + weights_key_override=None, + mask_path=None, + nside=None, + overwrite_config=False, + ): + """Compute effective survey statistics for a catalog version. + + Parameters + ---------- + ver : str + Version string registered in the catalog config. + weights_key_override : str, optional + Override the weight column key (defaults to the configured `w_col`). + mask_path : str, optional + Explicit mask path to use when measuring survey area. + nside : int, optional + If provided, compute survey area from the catalog using this NSIDE when no + mask path is available. + overwrite_config : bool, optional + If True, persist the derived statistics back to the catalog configuration. + + Returns + ------- + dict + Dictionary containing: + - area_deg2: Survey area in square degrees. + - n_eff: Effective number density per arcmin^2. + - sigma_e: Per-component shape noise. + - sum_w: Sum of weights. + - sum_w2: Sum of squared weights. + - catalog_size: Number of galaxies processed. + """ + if ver not in self.cc: + raise KeyError(f"Version {ver} not found in catalog configuration") + + shear_cfg = self.cc[ver]["shear"] + cov_th = self.cc[ver].get("cov_th", {}) + + if "path" not in shear_cfg: + raise KeyError(f"No shear catalog path defined for version {ver}") + + catalog_path = shear_cfg["path"] + if not os.path.exists(catalog_path): + raise FileNotFoundError(f"Shear catalog not found: {catalog_path}") + + data = fits.getdata(catalog_path, memmap=True) + n_rows = len(data) + + e1 = np.asarray(data[shear_cfg["e1_col"]], dtype=float) + e2 = np.asarray(data[shear_cfg["e2_col"]], dtype=float) + + weight_column = weights_key_override or shear_cfg["w_col"] + if weight_column not in data.columns.names: + raise KeyError(f"Weight column '{weight_column}' missing in {catalog_path}") + + w = np.asarray(data[weight_column], dtype=float) + + sum_w = float(np.sum(w)) + sum_w2 = float(np.sum(w**2)) + sum_w2_e2 = float(np.sum((w**2) * (e1**2 + e2**2))) + + if mask_path is not None: + if not os.path.exists(mask_path): + raise FileNotFoundError(f"Mask path not found: {mask_path}") + mask_candidate = mask_path + else: + mask_candidate = self.cc[ver].get("mask") + if isinstance(mask_candidate, str) and not os.path.isabs(mask_candidate): + mask_candidate = str(Path(self.cc[ver]["subdir"]) / mask_candidate) + if mask_candidate is not None and not os.path.exists(mask_candidate): + mask_candidate = None + + area_deg2 = None + if mask_candidate is not None and os.path.exists(mask_candidate): + area_deg2 = self._area_from_mask(mask_candidate) + elif cov_th.get("A") is not None: + area_deg2 = float(cov_th["A"]) + elif nside is not None: + area_deg2 = self._area_from_catalog(catalog_path, nside) + else: + raise ValueError( + f"Unable to determine survey area for {ver}. Provide mask_path or nside." + ) + + area_arcmin2 = area_deg2 * 3600.0 + + n_eff = (sum_w**2) / (area_arcmin2 * sum_w2) if sum_w2 > 0 else 0.0 + sigma_e = np.sqrt(sum_w2_e2 / sum_w2) if sum_w2 > 0 else 0.0 + + results = { + "area_deg2": area_deg2, + "n_eff": n_eff, + "sigma_e": sigma_e, + "sum_w": sum_w, + "sum_w2": sum_w2, + "catalog_size": n_rows, + } + + if overwrite_config: + if "cov_th" not in self.cc[ver]: + self.cc[ver]["cov_th"] = {} + self.cc[ver]["cov_th"]["A"] = float(area_deg2) + self.cc[ver]["cov_th"]["n_e"] = float(n_eff) + self.cc[ver]["cov_th"]["sigma_e"] = float(sigma_e) + self._write_catalog_config() + + return results + + def _area_from_catalog(self, catalog_path, nside): + data = fits.getdata(catalog_path, memmap=True) + ra = np.asarray(data["RA"], dtype=float) + dec = np.asarray(data["Dec"], dtype=float) + theta = np.radians(90.0 - dec) + phi = np.radians(ra) + pix = hp.ang2pix(nside, theta, phi, lonlat=False) + unique_pix = np.unique(pix) + return float(unique_pix.size * hp.nside2pixarea(nside, degrees=True)) + + def _area_from_mask(self, mask_map_path): + mask = hp.read_map(mask_map_path, dtype=np.float64) + return float(mask.sum() * hp.nside2pixarea(hp.get_nside(mask), degrees=True)) + + def _write_catalog_config(self): + with self.catalog_config_path.open("w") as file: + yaml.dump(self.cc, file, sort_keys=False) + def color_reset(self): print(colorama.Fore.BLACK, end="") diff --git a/src/sp_validation/tests/test_catalog_paths.py b/src/sp_validation/tests/test_catalog_paths.py index a66778c..51e4322 100644 --- a/src/sp_validation/tests/test_catalog_paths.py +++ b/src/sp_validation/tests/test_catalog_paths.py @@ -17,22 +17,6 @@ Path(__file__).resolve().parents[3] / "notebooks" / "cosmo_val" / "cat_config.yaml" ) -EXPECTED_MISSING = { - "SP_v1.0": {"shear", "psf"}, - "SP_v1.1": {"shear", "psf"}, - "SP_v1.4": {"shear", "star", "psf"}, - "SP_v1.4_conv": {"shear", "star", "psf"}, - "SP_v1.4_noalpha": {"shear", "star", "psf"}, - "SP_v1.4-P1+3": {"shear", "star", "psf"}, - "SP_v1.4-P1+3_wcs": {"shear", "star", "psf"}, - "SP_v1.4-P1+3_no_alpha": {"shear", "star", "psf"}, - "SP_v1.4-P1+3_li_2024": {"shear", "star", "psf"}, - "SP_v1.4-P1+3+4": {"shear", "star", "psf"}, - "SP_v1.4-P1+3+4_wcs": {"shear", "star", "psf"}, - "SP_v1.4-P1+3+4_no_alpha": {"shear", "star", "psf"}, - "SP_matched_MP_v1.0": {"shear"}, -} - def _resolve(base: Path, candidate: str) -> Path: """Return an absolute path given a base directory and a candidate string.""" @@ -75,23 +59,7 @@ def test_catalog_files_exist(): if not resolved_path.is_file(): missing[name].add(block_name) - unexpected = { - name: blocks - EXPECTED_MISSING.get(name, set()) - for name, blocks in missing.items() - if blocks - EXPECTED_MISSING.get(name, set()) - } - assert not unexpected, ( - "Unexpected missing catalog files detected: " - f"{ {name: sorted(blocks) for name, blocks in unexpected.items()} }" - ) - - resolved = { - name: expected - missing.get(name, set()) - for name, expected in EXPECTED_MISSING.items() - if expected - missing.get(name, set()) - } - assert not resolved, ( - "Previously missing catalog files are now available; " - f"update EXPECTED_MISSING to reflect this change: " - f"{ {name: sorted(blocks) for name, blocks in resolved.items()} }" + assert not missing, ( + "Catalog configuration references missing files: " + f"{ {name: sorted(blocks) for name, blocks in missing.items()} }" ) From 1b552381ca2fc5e63fad24835c11138e49fe5239 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 15 Oct 2025 11:41:43 +0200 Subject: [PATCH 08/34] add docstring --- src/sp_validation/cosmo_val.py | 86 ++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 4527532..e93d95b 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -24,6 +24,92 @@ # %% class CosmologyValidation: + """Framework for cosmic shear validation and systematics analysis. + + Handles two-point correlation function measurements, PSF systematics (rho/tau), + pseudo-C_ell analysis, and covariance estimation for weak lensing surveys. + Supports multiple catalog versions with automatic leakage-corrected variants. + + Parameters + ---------- + versions : list of str + Catalog version identifiers to analyze. Appending '_leak_corr' to a base + version creates a virtual catalog using leakage-corrected ellipticity columns + (e1_col_corrected/e2_col_corrected) from the base version configuration. + catalog_config : str, default './cat_config.yaml' + Path to catalog configuration YAML defining survey metadata, file paths, + and analysis settings for each version. + output_dir : str, optional + Override for output directory. If None, uses catalog config's paths.output. + rho_tau_method : {'lsq', 'mcmc'}, default 'lsq' + Fitting method for PSF leakage systematics parameters. + cov_estimate_method : {'th', 'jk'}, default 'th' + Covariance estimation: 'th' for semi-analytic theory, 'jk' for jackknife. + compute_cov_rho : bool, default True + Whether to compute covariance for rho statistics during PSF analysis. + n_cov : int, default 100 + Number of realizations for covariance estimation when using theory method. + theta_min : float, default 0.1 + Minimum angular separation in arcminutes for correlation function binning. + theta_max : float, default 250 + Maximum angular separation in arcminutes for correlation function binning. + nbins : int, default 20 + Number of angular bins for TreeCorr real-space correlation functions. + var_method : {'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'}, default 'jackknife' + TreeCorr variance estimation method. + npatch : int, default 20 + Number of spatial patches for jackknife variance estimation. + quantile : float, default 0.1587 + Quantile for uncertainty bands in plots (default: 1-sigma ≈ 0.159). + theta_min_plot : float, default 0.08 + Minimum angular scale for plotting (may differ from analysis cut). + theta_max_plot : float, default 250 + Maximum angular scale for plotting. + ylim_alpha : list of float, default [-0.005, 0.05] + Y-axis limits for alpha systematic parameter plots. + ylim_xi_sys_ratio : list of float, default [-0.02, 0.5] + Y-axis limits for xi systematics ratio plots. + nside : int, default 1024 + HEALPix resolution for pseudo-C_ell analysis and area computation. + binning : {'powspace', 'linspace', 'logspace'}, default 'powspace' + Ell binning scheme for pseudo-C_ell (powspace = ell^power spacing). + power : float, default 0.5 + Exponent for power-law binning when binning='powspace'. + n_ell_bins : int, default 32 + Number of ell bins for pseudo-C_ell analysis. + pol_factor : bool, default True + Apply polarization correction factor in pseudo-C_ell calculations. + nrandom_cell : int, default 10 + Number of random realizations for C_ell error estimation. + cosmo_params : dict, optional + Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. + redshift_file : str, optional + Path to n(z) file for theory calculations. Format: columns of (z, n(z)). + + Attributes + ---------- + versions : list of str + Validated catalog versions after processing _leak_corr variants. + cc : dict + Loaded catalog configuration with resolved absolute paths. + catalog_config_path : Path + Resolved path to the catalog configuration file. + treecorr_config : dict + Configuration dictionary passed to TreeCorr correlation objects. + cosmo : pyccl.Cosmology + Cosmology object for theory predictions. + z_dist : ndarray or None + Redshift distribution loaded from redshift_file if provided. + + Notes + ----- + - Path resolution: Relative paths in catalog config are resolved using each + version's 'subdir' field as the base directory. + - Virtual _leak_corr versions: These create deep copies of the base version + config, swapping e1_col/e2_col with e1_col_corrected/e2_col_corrected. + - TreeCorr cross_patch_weight: Automatically set to 'match' for jackknife, + 'simple' otherwise, following TreeCorr best practices. + """ def __init__( self, versions, From 2a6a3e2c51ffc5f830b0fc5eb8dcde31eea890e7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 18:38:12 +0200 Subject: [PATCH 09/34] refactor: remove redshift_file parameter and use catalog config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eliminate redundant redshift_file parameter and load n(z) directly from catalog configuration. Added get_redshift() method as single source of truth. - Renamed shear.redshift_distr → shear.redshift_path in cat_config.yaml - Added get_redshift(version) method for catalog-aware n(z) loading - Updated calculate_pure_eb(), plot_pure_eb(), calculate_pseudo_cl_eb_cov() to use get_redshift() - Removed redshift_file parameter from __init__, calculate_pure_eb(), plot_pure_eb() signatures and docstrings 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- ..._minsep=1_maxsep=250_nbins=20_npatch=1.ini | 124 ++++++++++++++++++ ..._minsep=1_maxsep=250_nbins=20_npatch=1.ini | 124 ++++++++++++++++++ notebooks/cosmo_val/cat_config.yaml | 20 +-- src/sp_validation/cosmo_val.py | 50 +++---- 4 files changed, 279 insertions(+), 39 deletions(-) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini new file mode 100644 index 0000000..6be956e --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +FITS_FILE = data/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +priors = cosmosis_config/priors_psf.ini +values = cosmosis_config/values_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/test_new_pipeline + +[polychord] +polychord_outfile_root = SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +add_xi_sys=T +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini new file mode 100644 index 0000000..6bc3a7d --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +FITS_FILE = data/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +priors = cosmosis_config/priors_psf.ini +values = cosmosis_config/values_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/test_new_pipeline + +[polychord] +polychord_outfile_root = SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +add_xi_sys=T +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index dd02dfb..cf9e89e 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -211,7 +211,7 @@ SP_test: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_iv e1_col: e1 e1_PSF_col: e1_PSF @@ -883,7 +883,7 @@ SP_v1.4.5: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -928,7 +928,7 @@ SP_v1.4.5.A: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: shapepipe_SPv1.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: g1 e1_PSF_col: e1_PSF @@ -971,7 +971,7 @@ SP_v1.4.5_bright: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_PSF_col: e1_PSF @@ -1014,7 +1014,7 @@ SP_v1.4.5_faint: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_PSF_col: e1_PSF @@ -1057,7 +1057,7 @@ SP_v1.4.5_glass_mock: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: e1 e1_PSF_col: e1_PSF @@ -1100,7 +1100,7 @@ SP_v1.4.5_intermediate: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_PSF_col: e1_PSF @@ -1144,7 +1144,7 @@ SP_v1.4.6: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -1190,7 +1190,7 @@ SP_v1.4.7: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -1236,7 +1236,7 @@ SP_v1.4.8: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index e93d95b..aa97dac 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -83,8 +83,6 @@ class CosmologyValidation: Number of random realizations for C_ell error estimation. cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. - redshift_file : str, optional - Path to n(z) file for theory calculations. Format: columns of (z, n(z)). Attributes ---------- @@ -98,8 +96,6 @@ class CosmologyValidation: Configuration dictionary passed to TreeCorr correlation objects. cosmo : pyccl.Cosmology Cosmology object for theory predictions. - z_dist : ndarray or None - Redshift distribution loaded from redshift_file if provided. Notes ----- @@ -136,7 +132,6 @@ def __init__( pol_factor=True, nrandom_cell=10, cosmo_params=None, - redshift_file=None, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -167,8 +162,6 @@ def __init__( # Use Planck 2018 defaults self.cosmo = get_cosmo() - # load redshift distribution from file if provided - self.z_dist = np.loadtxt(redshift_file) if redshift_file is not None else None self.treecorr_config = { "ra_units": "degrees", @@ -241,6 +234,24 @@ def resolve_paths_for_version(ver): if not os.path.exists(cc["paths"]["output"]): os.mkdir(cc["paths"]["output"]) + def get_redshift(self, version): + """Load redshift distribution for a catalog version. + + Parameters + ---------- + version : str + Catalog version identifier + + Returns + ------- + z : ndarray + Redshift values + nz : ndarray + n(z) probability density + """ + redshift_path = self.cc[version]["shear"]["redshift_path"] + return np.loadtxt(redshift_path, unpack=True) + def compute_survey_stats( self, ver, @@ -1684,7 +1695,6 @@ def calculate_pure_eb( var_method="jackknife", cov_path_int=None, cosmo_cov=None, - redshift_file=None, n_samples=1000, ): """ @@ -1782,18 +1792,8 @@ def calculate_pure_eb( # Get redshift distribution if using analytic covariance if cov_path_int is not None: - if redshift_file is None: - try: - print("Inheriting redshift distribution from self.z_dist") - z_dist = self.z_dist - except AttributeError: - raise ValueError( - "redshift distribution must be provided either to this function" - " or to the CosmologyValidation class upon creation " - "if using an analytic covariance." - ) - else: - z_dist = np.loadtxt(redshift_file) + z, nz = self.get_redshift(version) + z_dist = np.column_stack([z, nz]) else: z_dist = None @@ -1826,7 +1826,6 @@ def plot_pure_eb( var_method="jackknife", cov_path_int=None, cosmo_cov=None, - redshift_file=None, n_samples=1000, results=None, **kwargs @@ -1864,8 +1863,6 @@ def plot_pure_eb( Path to integration covariance matrix for semi-analytical calculation cosmo_cov : pyccl.Cosmology, optional Cosmology for theoretical predictions in semi-analytical covariance - redshift_file : str, optional - Path to redshift distribution file for semi-analytical covariance n_samples : int Number of Monte Carlo samples for semi-analytical covariance (default: 1000) results : dict or list, optional @@ -1953,7 +1950,6 @@ def plot_pure_eb( var_method=var_method, cov_path_int=cov_path_int, cosmo_cov=cosmo_cov, - redshift_file=redshift_file, n_samples=n_samples, ) @@ -2310,14 +2306,10 @@ def calculate_pseudo_cl_eb_cov(self): self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") lmax = 2 * self.nside - redshift_path = Path(self.cc[ver]["shear"]["redshift_distr"]) - if not redshift_path.is_absolute(): - redshift_path = Path(self.cc[ver]["subdir"]) / redshift_path - path_redshift_distr = str(redshift_path) + z, dndz = self.get_redshift(ver) pw = hp.pixwin(nside, lmax=lmax) # Load redshift distribution and calculate theory C_ell - z, dndz = np.loadtxt(path_redshift_distr, unpack=True) ell = np.arange(1, lmax + 1) fiducial_cl = ( get_theo_c_ell( From 419212bc33cd87e072075df0e1f0e612ebb90908 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 18:55:55 +0200 Subject: [PATCH 10/34] fix: remove obsolete data_base_dir parameter from eb_plots.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update CosmologyValidation call to match current API. The data_base_dir parameter was removed in the catalog config refactor; all paths are now resolved from cat_config.yaml. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py index e0b371c..5d7b740 100644 --- a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py +++ b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py @@ -48,7 +48,6 @@ cv = CosmologyValidation( versions=[config["version"]], - data_base_dir="/n17data/mkilbing/astro/data", ) # %% eb_results = cv.calculate_pure_eb(version=config["version"], **config["pure_eb"]) From a14123cf19cefa6a3dc8b47b41ffa21ce0b304b7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 22:09:41 +0200 Subject: [PATCH 11/34] feat: add glass mock seed variants and v1.4.6 glass mock catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for seed-specific mock catalog variants (e.g., SP_v1.4.5_glass_mock_seed1) by extracting and substituting seed tokens in shear paths. Enables exploring multiple random realizations of the same mock survey. - Add SP_v1.4.6_glass_mock catalog entry with v1.4.6 survey specs - Refactor version processing in __init__ to use recursive ensure_version_exists() - Support _seed variants that deep-copy base config and substitute seed token - Handle _seed_leak_corr combinations by materializing seed config first - Add explicit error checking for missing seed tokens in paths - Add regression tests for seed variant creation and error cases Seed variant examples: - SP_v1.4.5_glass_mock_seed1 → unions_glass_sim_00001_4096.fits - SP_v1.4.6_glass_mock_seed12 → unions_glass_sim_00012_4096.fits - SP_v1.4.5_glass_mock_seed1_leak_corr → combines both transforms 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- notebooks/cosmo_val/cat_config.yaml | 44 +++++++++++ src/sp_validation/cosmo_val.py | 95 +++++++++++++++++------ src/sp_validation/tests/test_cosmo_val.py | 91 ++++++++++++++++++++++ 3 files changed, 207 insertions(+), 23 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index cf9e89e..71f145c 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1070,6 +1070,50 @@ SP_v1.4.5_glass_mock: e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.6_glass_mock: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkgreen + getdist_colour: 0.0, 0.7, 0.0 + ls: dashed + marker: d + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits + SP_v1.4.5_intermediate: subdir: /n17data/murray/unions_cats pipeline: SP diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index aa97dac..18dcf71 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,6 +1,7 @@ # %% import copy import os +import re from pathlib import Path import colorama @@ -106,6 +107,34 @@ class CosmologyValidation: - TreeCorr cross_patch_weight: Automatically set to 'match' for jackknife, 'simple' otherwise, following TreeCorr best practices. """ + + _SEED_SUFFIX_RE = re.compile(r"^(?P.+)_seed(?P\d+)$") + _SEED_TOKEN_RE = re.compile(r"seed(?P[_-]?)(?P\d+)") + + @classmethod + def _split_seed_variant(cls, version): + """Return the base version and seed label if version encodes a seed.""" + match = cls._SEED_SUFFIX_RE.match(version) + if match is None: + return None, None + return match.group("base"), match.group("seed") + + @classmethod + def _apply_seed_token(cls, path, seed_value, version, base_version, catalog_config): + """Replace the final seed token in the path with the requested seed value.""" + matches = list(cls._SEED_TOKEN_RE.finditer(path)) + if not matches: + raise ValueError( + f"Cannot materialize '{version}': shear path '{path}' for base version " + f"'{base_version}' does not contain a 'seed' token. " + f"Update {catalog_config} or drop the seed suffix." + ) + match = matches[-1] + sep = match.group("sep") or "" + replacement = f"seed{sep}{seed_value}" + start, end = match.span() + return f"{path[:start]}{replacement}{path[end:]}" + def __init__( self, versions, @@ -191,38 +220,58 @@ def resolve_paths_for_version(ver): resolve_paths_for_version("nz") processed = {"nz"} final_versions = [] + leak_suffix = "_leak_corr" - for ver in versions: - if ver.endswith("_leak_corr"): - base_ver = ver.replace("_leak_corr", "") - target = base_ver - if base_ver not in cc: - raise KeyError( - f"Base version {base_ver} not found for {ver} in config file " - f"{catalog_config}" - ) - if "e1_col_corrected" not in cc[base_ver]["shear"]: + def ensure_version_exists(ver): + if ver in processed: + return + + if ver in cc: + resolve_paths_for_version(ver) + processed.add(ver) + return + + seed_base, seed_label = self._split_seed_variant(ver) + + if ver.endswith(leak_suffix): + base_ver = ver[: -len(leak_suffix)] + ensure_version_exists(base_ver) + shear_cfg = cc[base_ver]["shear"] + if "e1_col_corrected" not in shear_cfg or "e2_col_corrected" not in shear_cfg: raise ValueError( f"{base_ver} does not have e1_col_corrected/e2_col_corrected " f"fields; cannot create {ver}" ) - else: - target = ver if ver not in cc: - raise KeyError( - f"Version string {ver} not found in config file " - f"{catalog_config}" + cc[ver] = copy.deepcopy(cc[base_ver]) + cc[ver]["shear"]["e1_col"] = shear_cfg["e1_col_corrected"] + cc[ver]["shear"]["e2_col"] = shear_cfg["e2_col_corrected"] + resolve_paths_for_version(ver) + processed.add(ver) + return + + if seed_base is not None: + ensure_version_exists(seed_base) + if ver not in cc: + cc[ver] = copy.deepcopy(cc[seed_base]) + seed_path = self._apply_seed_token( + cc[seed_base]["shear"]["path"], + seed_label, + ver, + seed_base, + catalog_config, ) + cc[ver]["shear"]["path"] = seed_path + resolve_paths_for_version(ver) + processed.add(ver) + return - if target not in processed: - resolve_paths_for_version(target) - processed.add(target) - - if ver.endswith("_leak_corr"): - cc[ver] = copy.deepcopy(cc[base_ver]) - cc[ver]["shear"]["e1_col"] = cc[base_ver]["shear"]["e1_col_corrected"] - cc[ver]["shear"]["e2_col"] = cc[base_ver]["shear"]["e2_col_corrected"] + raise KeyError( + f"Version string {ver} not found in config file {catalog_config}" + ) + for ver in versions: + ensure_version_exists(ver) final_versions.append(ver) self.versions = final_versions diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 55b8f80..16a4c48 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -11,6 +11,7 @@ import os import pytest +import yaml from sp_validation.cosmo_val import CosmologyValidation @@ -42,6 +43,57 @@ def base_config(self, tmp_path): "nbins": 20, } + @staticmethod + def _make_seed_config(tmp_path, shear_filename): + """Create a minimal catalog config for seed variant testing.""" + base_version = "TestCatalog" + base_dir = tmp_path / "catalog" + base_dir.mkdir() + (base_dir / shear_filename).touch() + + star_filename = "star_seed_1234.fits" + (base_dir / star_filename).touch() + + nz_dir = tmp_path / "nz" + nz_dir.mkdir() + (nz_dir / "dndz.txt").write_text("0.1 1.0\n") + + output_dir = tmp_path / "output" + output_dir.mkdir() + + config_path = tmp_path / "seed_config.yaml" + config_data = { + "nz": { + "subdir": str(nz_dir), + "dndz": {"blind": "A", "path": "dndz.txt"}, + }, + "paths": {"output": str(output_dir)}, + base_version: { + "subdir": str(base_dir), + "pipeline": "SP", + "shear": { + "path": shear_filename, + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "e1_col_corrected": "e1_corr", + "e2_col_corrected": "e2_corr", + }, + "star": {"path": star_filename}, + }, + } + config_path.write_text(yaml.dump(config_data, sort_keys=False)) + + params = { + "catalog_config": str(config_path), + "output_dir": str(output_dir), + "npatch": 1, + "theta_min": 1.0, + "theta_max": 250.0, + "nbins": 20, + } + return params, base_version + @pytest.mark.parametrize( "version,e1_col,e2_col", [ @@ -125,3 +177,42 @@ def test_additive_bias_leak_corrected_columns(self, base_config, version): # Verify the values are numeric assert isinstance(cv.c1[version_leak_corr], float) assert isinstance(cv.c2[version_leak_corr], float) + + def test_seed_variant_updates_shear_path(self, tmp_path): + """Seeded versions should materialize a seed-specific shear path.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_seed_1234.fits" + ) + seed_version = f"{base_version}_seed007" + + cv = CosmologyValidation(versions=[seed_version], **params) + + assert cv.versions == [seed_version] + assert seed_version in cv.cc + assert cv.cc[seed_version]["shear"]["path"].endswith("shear_seed_007.fits") + + def test_seed_leak_corr_materializes_seed_first(self, tmp_path): + """_seed_leak_corr should clone the seed variant before leak fixes.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_seed_1234.fits" + ) + leak_version = f"{base_version}_seed007_leak_corr" + seed_version = f"{base_version}_seed007" + + cv = CosmologyValidation(versions=[leak_version], **params) + + assert cv.versions == [leak_version] + assert seed_version in cv.cc + assert cv.cc[seed_version]["shear"]["path"].endswith("shear_seed_007.fits") + assert cv.cc[leak_version]["shear"]["e1_col"] == "e1_corr" + assert cv.cc[leak_version]["shear"]["e2_col"] == "e2_corr" + + def test_seed_variant_without_token_errors(self, tmp_path): + """Missing seed token in shear path should raise a descriptive error.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_base.fits" + ) + seed_version = f"{base_version}_seed123" + + with pytest.raises(ValueError, match="seed"): + CosmologyValidation(versions=[seed_version], **params) From 8d14a83d8453542ac83481c490ad534b7e89c202 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 22:14:45 +0200 Subject: [PATCH 12/34] test: add v1.4.6 glass mock to base tests and seed variant test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add SP_v1.4.6_glass_mock to parametrized additive bias tests - Update SP_v1.4.6_glass_mock shear path to glass_mock_v1.4.6 directory - Add test_v1_4_6_glass_mock_seed_variant to verify seed9 variant loads correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- notebooks/cosmo_val/cat_config.yaml | 2 +- src/sp_validation/tests/test_cosmo_val.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 71f145c..bcc4274 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1100,7 +1100,7 @@ SP_v1.4.6_glass_mock: shear: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00000_4096.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: e1 diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 16a4c48..0087b6d 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -100,6 +100,7 @@ def _make_seed_config(tmp_path, shear_filename): ("SP_v1.4.5", "e1", "e2"), ("SP_v1.4.6", "e1", "e2"), ("SP_v1.4.5_glass_mock", "e1", "e2"), + ("SP_v1.4.6_glass_mock", "e1", "e2"), ("SP_v1.4.5_bright", "e1", "e2"), ("SP_v1.4.5_faint", "e1", "e2"), ("SP_v1.4.5_intermediate", "e1", "e2"), @@ -216,3 +217,24 @@ def test_seed_variant_without_token_errors(self, tmp_path): with pytest.raises(ValueError, match="seed"): CosmologyValidation(versions=[seed_version], **params) + + def test_v1_4_6_glass_mock_seed_variant(self, base_config): + """Test that v1.4.6 glass mock seed variant loads with correct path.""" + seed = 9 + seed_version = f"SP_v1.4.6_glass_mock_seed{seed}" + + cv = CosmologyValidation( + versions=[seed_version], + **base_config, + ) + + # Verify version was created + assert cv.versions == [seed_version] + assert seed_version in cv.cc + + # Verify seed was substituted in shear path + expected_filename = f"unions_glass_sim_{seed:05d}_4096.fits" + assert expected_filename in cv.cc[seed_version]["shear"]["path"] + + # Verify path points to v1.4.6 glass mock directory + assert "glass_mock_v1.4.6" in cv.cc[seed_version]["shear"]["path"] From 60ffcd3e45d67cde114d9040f79e6d61d146f558 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 23:01:58 +0200 Subject: [PATCH 13/34] refactor: use path templates for elegant seed variant handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace brittle regex-based seed token detection with config-driven path templates using Python string formatting. Each catalog specifies a path_template with {seed:05d} or {seed} placeholders. - Add path_template fields to SP_v1.4.5_glass_mock and SP_v1.4.6_glass_mock - Simplify _split_seed_variant to pure string operations (no regex) - Add _materialize_seed_path using .format() for clean templating - Fallback to legacy seed-token extraction if no template provided - All 15 tests pass including new v1.4.6 glass mock seed9 test Benefits: - Per-catalog control of path formatting without code changes - No complex regex fragility - Clear, self-documenting config entries 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- notebooks/cosmo_val/cat_config.yaml | 4 +- src/sp_validation/cosmo_val.py | 86 +++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index bcc4274..219f054 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1057,6 +1057,7 @@ SP_v1.4.5_glass_mock: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + path_template: /n09data/guerrini/glass_mock/results/unions_glass_sim_{seed:05d}_4096.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: e1 @@ -1100,7 +1101,8 @@ SP_v1.4.6_glass_mock: shear: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00000_4096.fits + path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00001_4096.fits + path_template: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_{seed:05d}_4096.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: e1 diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 18dcf71..07ded12 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,7 +1,6 @@ # %% import copy import os -import re from pathlib import Path import colorama @@ -108,32 +107,73 @@ class CosmologyValidation: 'simple' otherwise, following TreeCorr best practices. """ - _SEED_SUFFIX_RE = re.compile(r"^(?P.+)_seed(?P\d+)$") - _SEED_TOKEN_RE = re.compile(r"seed(?P[_-]?)(?P\d+)") - - @classmethod - def _split_seed_variant(cls, version): + @staticmethod + def _split_seed_variant(version): """Return the base version and seed label if version encodes a seed.""" - match = cls._SEED_SUFFIX_RE.match(version) - if match is None: + if "_seed" not in version: + return None, None + base, seed_label = version.rsplit("_seed", 1) + if not base or not seed_label.isdigit(): return None, None - return match.group("base"), match.group("seed") + return base, seed_label - @classmethod - def _apply_seed_token(cls, path, seed_value, version, base_version, catalog_config): - """Replace the final seed token in the path with the requested seed value.""" - matches = list(cls._SEED_TOKEN_RE.finditer(path)) - if not matches: + @staticmethod + def _materialize_seed_path( + base_cfg, seed_label, version, base_version, catalog_config + ): + """Render the seed-specific shear path using Python string formatting.""" + shear_cfg = base_cfg["shear"] + template = shear_cfg.get("path_template") + + try: + seed_value = int(seed_label) + except ValueError as error: + raise ValueError( + f"Seed suffix for '{version}' is not numeric; cannot materialize path." + ) from error + + format_context = {"seed": seed_value, "seed_label": seed_label} + + if template: + try: + return template.format(**format_context) + except KeyError as error: + raise KeyError( + f"Missing placeholder '{error.args[0]}' in path_template for " + f"'{base_version}' while materializing '{version}'. Update " + f"{catalog_config}." + ) from error + except ValueError as error: + raise ValueError( + f"Invalid format specification in path_template for '{base_version}' " + f"while materializing '{version}'." + ) from error + + path = shear_cfg.get("path", "") + token_start = path.rfind("seed") + if token_start == -1: + raise ValueError( + f"Cannot materialize '{version}': '{base_version}' lacks a shear " + f"path_template and its shear path '{path}' does not contain a 'seed' " + f"token. Update {catalog_config}." + ) + cursor = token_start + 4 # len("seed") + if cursor < len(path) and not path[cursor].isdigit(): + cursor += 1 + digit_start = cursor + while cursor < len(path) and path[cursor].isdigit(): + cursor += 1 + digit_end = cursor + digits = path[digit_start:digit_end] + if not digits: raise ValueError( f"Cannot materialize '{version}': shear path '{path}' for base version " - f"'{base_version}' does not contain a 'seed' token. " - f"Update {catalog_config} or drop the seed suffix." + f"'{base_version}' lacks digits after the seed token. Update " + f"{catalog_config}." ) - match = matches[-1] - sep = match.group("sep") or "" - replacement = f"seed{sep}{seed_value}" - start, end = match.span() - return f"{path[:start]}{replacement}{path[end:]}" + + template = f"{path[:digit_start]}{{seed_label}}{path[digit_end:]}" + return template.format(**format_context) def __init__( self, @@ -254,8 +294,8 @@ def ensure_version_exists(ver): ensure_version_exists(seed_base) if ver not in cc: cc[ver] = copy.deepcopy(cc[seed_base]) - seed_path = self._apply_seed_token( - cc[seed_base]["shear"]["path"], + seed_path = self._materialize_seed_path( + cc[seed_base], seed_label, ver, seed_base, From 8957ce2756622eeada1b5fc335d0e6ce5d370f6f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 17 Oct 2025 13:59:26 +0200 Subject: [PATCH 14/34] test: add v1.4.6 glass mock default seed test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verify that SP_v1.4.6_glass_mock without a seed suffix correctly uses the default seed 00001 from the configured path field. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/sp_validation/tests/test_cosmo_val.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 0087b6d..42759f8 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -238,3 +238,19 @@ def test_v1_4_6_glass_mock_seed_variant(self, base_config): # Verify path points to v1.4.6 glass mock directory assert "glass_mock_v1.4.6" in cv.cc[seed_version]["shear"]["path"] + + def test_v1_4_6_glass_mock_default_seed(self, base_config): + """Test that glass mock without seed suffix uses the default seed_00001.""" + cv = CosmologyValidation( + versions=["SP_v1.4.6_glass_mock"], + **base_config, + ) + + # Verify version loads without seed suffix + assert cv.versions == ["SP_v1.4.6_glass_mock"] + assert "SP_v1.4.6_glass_mock" in cv.cc + + # Verify it uses the default path (seed_00001 for v1.4.6) + path = cv.cc["SP_v1.4.6_glass_mock"]["shear"]["path"] + assert "unions_glass_sim_00001_4096.fits" in path + assert "glass_mock_v1.4.6" in path From bdc14617cc0c66cc35ce67ec81590f328ed2f3f0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 23 Oct 2025 13:53:19 +0200 Subject: [PATCH 15/34] consistent b-mode ylims --- src/sp_validation/b_modes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index d24f9e1..09ce030 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -659,6 +659,7 @@ def plot_pure_eb_correlations( ax.set(xscale="log", xlabel=r"$\theta$ [arcmin]") ax.axhline(0, alpha=0.3, color="k", linestyle="--", linewidth=0.5) ax.legend(loc="upper left") + ax.set_ylim(-0.5, 2) # Save the axis limits after data plotting but before adding gray regions original_xlims = [ax.get_xlim() for ax in axs] @@ -1037,6 +1038,7 @@ def plot_cosebis_modes( plt.legend() plt.xlabel("n (mode)") plt.ylabel("E_n, B_n") + plt.ylim(-0.5e-10, 3e-10) # Add scale cut information to title - use actual scale cut from results scale_info = "" @@ -1092,4 +1094,3 @@ def plot_cosebis_covariance_matrix(results, version, var_method, output_path): ax.set_title(f"{version} COSEBIs E/B {var_method} correlation matrix") plt.savefig(output_path, dpi=300, bbox_inches="tight") - From 9d3dac86b0b95d91b147cab3d2f3fadd0284155c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 24 Oct 2025 17:24:50 +0200 Subject: [PATCH 16/34] refactor: remove non-functional catalog versions and consolidate tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove 14 non-functional catalog versions from cat_config.yaml that have missing or misconfigured file paths: - LF_matched_SP_v1.0, LF_v1.0, LF_v2.0 - SP_matched_LF_v1.0, SP_v1.0_LFmask_4k, SP_v1.0_LFmask_8k - SP_v1.3_LFmask variants (4k, 8k, F2, SN7, SN8, li_2024, no_alpha) - SP_v1.4-P3_LFmask Retain 2 working LFmask versions: SP_v1.4_LFmask_8k and SP_v1.4_LFmask_8k_noalpha Consolidate test_catalog_paths.py functionality into test_cosmo_val.py: - test_catalog_paths_exist() now programmatically discovers all catalog versions - Simplify test_additive_bias_base_columns() to test only SP_v1.4.5 - Update test_additive_bias_leak_corrected_columns() to test SP_v1.4.6_leak_corr - Result: 10x faster test suite (614s → 25s) All tests pass. Remaining: 21 working catalog versions. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- notebooks/cosmo_val/cat_config.yaml | 539 ------------------ src/sp_validation/tests/test_catalog_paths.py | 65 --- src/sp_validation/tests/test_cosmo_val.py | 113 +++- 3 files changed, 90 insertions(+), 627 deletions(-) delete mode 100644 src/sp_validation/tests/test_catalog_paths.py diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 219f054..bb791ef 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -44,65 +44,6 @@ DES: e1_col: obs_e1 e2_col: obs_e2 path: psf_y3a1-v29.fits - -LF_matched_SP_v1.0: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP - pipeline: LF - colour: magenta - getdist_colour: 0.0, 0.5, 0.0 - ls: dashdot - marker: ^ - shear: - R: 1.0 - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - path: masked_lensfit_goldshape_2022v1.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: full_starcat_tmp_1_10_seed_1234.fits - -LF_v1.0: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/Lensfit - pipeline: LF - colour: r - getdist_colour: 1.0, 0.5, 0.0 - ls: solid - marker: '8' - shear: - R: 1.0 - covmat_file: ./covs/lensfit_A/cov_lensfit_A.txt - path: lensfit_goldshape_2022v1.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: full_starcat_tmp_1_10_seed_1234.fits - -LF_v2.0: - subdir: /n17data/mkilbing/astro/data/CFIS/v2.0/Lensfit - pipeline: LF - colour: r - ls: dashed - marker: p - shear: - R: 1.0 - path: lensfit_goldshape_2022v2_psf1.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1_psf - e2_col: e2_psf - path: Pall_hdu1_THELI_psfs1_mosaic.fits - SP_axel_v0.0: subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP @@ -147,40 +88,6 @@ SP_axel_v0.0: e1_col: E1_STAR_HSM e2_col: E2_STAR_HSM path: star_cat.fits - -SP_matched_LF_v1.0: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP - pipeline: SP - colour: magenta - getdist_colour: 0.90340530805574, 0.06198853997875753, 0.6527408678603205 - ls: dotted - marker: v - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - path: masked_matched_unions_shapepipe_extended_2022_v1.0.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - SP_test: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP @@ -223,7 +130,6 @@ SP_test: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.1.fits - SP_v0.1.1: subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP @@ -257,73 +163,6 @@ SP_v0.1.1: e1_col: E1_STAR_HSM e2_col: E2_STAR_HSM path: star_cat.fits - -SP_v1.0_LFmask_4k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: blue - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dashdot - marker: h - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - path: unions_shapepipe_2022_v1.0_mtheli4k.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - -SP_v1.0_LFmask_8k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: blue - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - path: unions_shapepipe_extended_2022_v1.0_mtheli8k.fits - e1_col: e1 - e1_PSF_col: e1_PSF - e2_col: e2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - SP_v1.3: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP @@ -361,324 +200,6 @@ SP_v1.3: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.0.3.fits - -SP_v1.3_LFmask_4k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: green - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dashdot - marker: h - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - path: unions_shapepipe_2022_v1.3_mtheli4k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - -SP_v1.3_LFmask_8k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: green - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_2022_v1.3_mtheli8k.fits - w_col: w - e1_col: e1 - e1_PSF_col: e1_PSF - e2_col: e2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - -SP_v1.3_LFmask_8k_F2: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: lawngreen - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 9.4 - n_psf: 0.48 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_2022_f2_v1.3_mtheli8k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - -SP_v1.3_LFmask_8k_SN7: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: purple - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 10.989 - n_psf: 0.48 - sigma_e: 0.3796 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_2022_sn7_v1.3_mtheli8k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - -SP_v1.3_LFmask_8k_SN8: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: darkgreen - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 9.7 - n_psf: 0.48 - sigma_e: 0.35 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_2022_sn8_v1.3_mtheli8k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - -SP_v1.3_LFmask_8k_li_2024: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: deepskyblue - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_rmalpha_li_2022_v1.3_mtheli8k.fits - w_col: w - e1_col: e1_cor_2 - e1_PSF_col: e1_PSF - e2_col: e2_cor_2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - -SP_v1.3_LFmask_8k_no_alpha: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: blueviolet - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_rmalpha_2022_v1.3_mtheli8k.fits - w_col: w - e1_col: e1_cor - e1_PSF_col: e1_PSF - e2_col: e2_cor - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - SP_v1.4-P3: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 pipeline: SP @@ -724,50 +245,6 @@ SP_v1.4-P3: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_P3_v1.4.fits - -SP_v1.4-P3_LFmask: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask/P3 - pipeline: SP - colour: black - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dashdot - marker: h - cov_th: - A: 223 - n_e: 7.44 - n_psf: 0.32 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.4-P3_LFmask.fits - path_tau: tau_stats_SP_v1.4-P3_LFmask.fits - path_xi_plus: xi_plus_SP_v1.4-P3_LFmask.fits - path_xi_minus: xi_minus_SP_v1.4-P3_LFmask.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.4_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - path: unions_shapepipe_2022_v1.4_mtheli4k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.4_mtheli4k.fits - SP_v1.4.1_noleakage: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP @@ -809,7 +286,6 @@ SP_v1.4.1_noleakage: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.1.fits - SP_v1.4.2: subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 pipeline: SP @@ -851,7 +327,6 @@ SP_v1.4.2: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -897,7 +372,6 @@ SP_v1.4.5: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5.A: subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 pipeline: SP @@ -940,7 +414,6 @@ SP_v1.4.5.A: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5_bright: subdir: /n17data/murray/unions_cats pipeline: SP @@ -983,7 +456,6 @@ SP_v1.4.5_bright: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5_faint: subdir: /n17data/murray/unions_cats pipeline: SP @@ -1026,7 +498,6 @@ SP_v1.4.5_faint: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5_glass_mock: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1070,7 +541,6 @@ SP_v1.4.5_glass_mock: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.6_glass_mock: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1115,7 +585,6 @@ SP_v1.4.6_glass_mock: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5_intermediate: subdir: /n17data/murray/unions_cats pipeline: SP @@ -1158,7 +627,6 @@ SP_v1.4.5_intermediate: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.6: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1204,7 +672,6 @@ SP_v1.4.6: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.7: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1250,7 +717,6 @@ SP_v1.4.7: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.8: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1296,7 +762,6 @@ SP_v1.4.8: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4_LFmask_8k: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP @@ -1343,7 +808,6 @@ SP_v1.4_LFmask_8k: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - SP_v1.4_LFmask_8k_noalpha: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP @@ -1390,7 +854,6 @@ SP_v1.4_LFmask_8k_noalpha: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - SP_v1.5.4: subdir: /n17data/UNIONS/WL/v1.5.x pipeline: SP @@ -1432,12 +895,10 @@ SP_v1.5.4: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.5.a.fits - nz: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz dndz: blind: A path: dndz - paths: output: ./output diff --git a/src/sp_validation/tests/test_catalog_paths.py b/src/sp_validation/tests/test_catalog_paths.py deleted file mode 100644 index 51e4322..0000000 --- a/src/sp_validation/tests/test_catalog_paths.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Lightweight checks that catalog paths referenced in ``cat_config.yaml`` exist. - -These tests do not load the data; they only ensure that the configuration points -to readable files so that downstream validation runs fail fast if a path drifts. -""" - -from __future__ import annotations - -from collections import defaultdict -from pathlib import Path -from typing import Dict, Iterator, Tuple - -import yaml - - -CATALOG_CONFIG = ( - Path(__file__).resolve().parents[3] / "notebooks" / "cosmo_val" / "cat_config.yaml" -) - - -def _resolve(base: Path, candidate: str) -> Path: - """Return an absolute path given a base directory and a candidate string.""" - candidate_path = Path(candidate) - return candidate_path if candidate_path.is_absolute() else base / candidate_path - - -def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: - """Yield (name, entry) pairs for catalog-like entries in the config.""" - for name, entry in config.items(): - if not isinstance(entry, dict): - continue - if "subdir" not in entry: - continue - yield name, entry - - -def test_catalog_files_exist(): - """Ensure every shear/star/psf entry references an on-disk file.""" - config = yaml.safe_load(CATALOG_CONFIG.read_text()) - - missing = defaultdict(set) - - for name, entry in _iter_catalog_entries(config): - base = Path(entry["subdir"]) - assert base.is_absolute(), f"{name}: subdir must be absolute ({base})" - - if name == "nz": - dndz_path = _resolve(base, entry["dndz"]["path"]) - assert ( - dndz_path.parent.is_dir() - ), f"{name}: dndz parent directory missing ({dndz_path.parent})" - continue - - for block_name in ("shear", "star", "psf"): - block = entry.get(block_name) - if not block: - continue - resolved_path = _resolve(base, block["path"]) - if not resolved_path.is_file(): - missing[name].add(block_name) - - assert not missing, ( - "Catalog configuration references missing files: " - f"{ {name: sorted(blocks) for name, blocks in missing.items()} }" - ) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 42759f8..afcd310 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -9,6 +9,9 @@ """ import os +from collections import defaultdict +from pathlib import Path +from typing import Dict, Iterator, Tuple import pytest import yaml @@ -94,28 +97,17 @@ def _make_seed_config(tmp_path, shear_filename): } return params, base_version - @pytest.mark.parametrize( - "version,e1_col,e2_col", - [ - ("SP_v1.4.5", "e1", "e2"), - ("SP_v1.4.6", "e1", "e2"), - ("SP_v1.4.5_glass_mock", "e1", "e2"), - ("SP_v1.4.6_glass_mock", "e1", "e2"), - ("SP_v1.4.5_bright", "e1", "e2"), - ("SP_v1.4.5_faint", "e1", "e2"), - ("SP_v1.4.5_intermediate", "e1", "e2"), - ("SP_v1.4.5.A", "g1", "g2"), - ("SP_v1.4.7", "e1", "e2"), - ("SP_v1.4.8", "e1", "e2"), - ], - ) - def test_additive_bias_base_columns(self, base_config, version, e1_col, e2_col): + def test_additive_bias_base_columns(self, base_config): """Test additive bias calculation using base ellipticity columns. This test initializes CosmologyValidation without an ellipticity_suffix, which means it will use the default columns defined in the catalog - configuration. + configuration. Tests SP_v1.4.5 with full additive bias computation. """ + version = "SP_v1.4.5" + e1_col = "e1" + e2_col = "e2" + cv = CosmologyValidation( versions=[version], **base_config, @@ -140,16 +132,16 @@ def test_additive_bias_base_columns(self, base_config, version, e1_col, e2_col): assert isinstance(cv.c1[version], float) assert isinstance(cv.c2[version], float) - @pytest.mark.parametrize("version", ["SP_v1.4.5"]) - def test_additive_bias_leak_corrected_columns(self, base_config, version): + def test_additive_bias_leak_corrected_columns(self, base_config): """Test additive bias calculation using leak-corrected columns. - This test requests a leak-corrected version by passing "{version}_leak_corr" + This test requests a leak-corrected version by passing "SP_v1.4.6_leak_corr" as the version name. The CosmologyValidation class automatically detects the _leak_corr suffix and creates the config entry using e1_col_corrected and e2_col_corrected from the base version. """ - version_leak_corr = f"{version}_leak_corr" + base_version = "SP_v1.4.6" + version_leak_corr = f"{base_version}_leak_corr" cv = CosmologyValidation( versions=[version_leak_corr], @@ -164,8 +156,8 @@ def test_additive_bias_leak_corrected_columns(self, base_config, version): assert cv.cc[version_leak_corr]["shear"]["e2_col"] == "e2_leak_corrected" # Verify original config entry remains unchanged - assert cv.cc[version]["shear"]["e1_col"] == "e1" - assert cv.cc[version]["shear"]["e2_col"] == "e2" + assert cv.cc[base_version]["shear"]["e1_col"] == "e1" + assert cv.cc[base_version]["shear"]["e2_col"] == "e2" # Calculate additive bias cv.calculate_additive_bias() @@ -179,6 +171,81 @@ def test_additive_bias_leak_corrected_columns(self, base_config, version): assert isinstance(cv.c1[version_leak_corr], float) assert isinstance(cv.c2[version_leak_corr], float) + @staticmethod + def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: + """Yield (name, entry) pairs for catalog-like entries in the config.""" + for name, entry in config.items(): + if not isinstance(entry, dict): + continue + if "subdir" not in entry: + continue + yield name, entry + + @staticmethod + def _resolve(base: Path, candidate: str) -> Path: + """Return an absolute path given a base directory and a candidate string.""" + candidate_path = Path(candidate) + return candidate_path if candidate_path.is_absolute() else base / candidate_path + + def test_catalog_paths_exist(self, base_config): + """Verify that catalog paths for active versions exist on disk. + + This is a lightweight test that checks that all files referenced in the + catalog configuration for UNIONS analysis versions actually exist. It + discovers versions programmatically from cat_config.yaml rather than + using hardcoded lists. + """ + # Get the path to catalog config + repo_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + ) + catalog_config_path = os.path.join( + repo_root, "notebooks", "cosmo_val", "cat_config.yaml" + ) + + config = yaml.safe_load(Path(catalog_config_path).read_text()) + + missing = defaultdict(set) + working = [] + nonfunctional = defaultdict(set) + + for version, entry in self._iter_catalog_entries(config): + # Skip nz entries and versions already tested in heavy tests + if version == "nz": + continue + + base = Path(entry["subdir"]) + version_missing = set() + + # Check shear, star, and psf files + for block_name in ("shear", "star", "psf"): + block = entry.get(block_name) + if not block: + continue + resolved_path = self._resolve(base, block["path"]) + if not resolved_path.is_file(): + version_missing.add(block_name) + + if version_missing: + nonfunctional[version] = version_missing + else: + working.append(version) + + # Print summary + print(f"\n✓ Working versions ({len(working)}):") + for v in sorted(working): + print(f" - {v}") + + if nonfunctional: + print(f"\n✗ Non-functional versions ({len(nonfunctional)}):") + for v in sorted(nonfunctional.keys()): + print(f" - {v}: missing {nonfunctional[v]}") + + assert not nonfunctional, ( + "Catalog configuration references missing files: " + f"{dict(nonfunctional)}" + ) + def test_seed_variant_updates_shear_path(self, tmp_path): """Seeded versions should materialize a seed-specific shear path.""" params, base_version = self._make_seed_config( From 27c0cbf26256a27fd3500353cb01d1bf40a0a553 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 27 Oct 2025 18:52:45 +0100 Subject: [PATCH 17/34] rework rho_tau.py to use consistent CosmologyValidation basename --- src/sp_validation/cosmo_val.py | 45 ++++- src/sp_validation/rho_tau.py | 357 ++++++++++++++++++++------------- 2 files changed, 260 insertions(+), 142 deletions(-) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index b00287f..d3a46ea 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -21,7 +21,11 @@ from .cosmology import get_cosmo, get_theo_c_ell from .plots import FootprintPlotter -from .rho_tau import get_params_rho_tau, get_rho_tau_w_cov, get_samples +from .rho_tau import ( + get_params_rho_tau, + get_rho_tau_w_cov, + get_samples, +) # %% @@ -593,6 +597,17 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise + def basename(self, version, treecorr_config=None, npatch=None): + cfg = treecorr_config or self.treecorr_config + patches = npatch or self.npatch + return ( + f"{version}_minsep={cfg['min_sep']}" + f"_maxsep={cfg['max_sep']}" + f"_nbins={cfg['nbins']}" + f"_npatch={patches}" + ) + + def calculate_rho_tau_stats(self): out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): @@ -600,13 +615,16 @@ def calculate_rho_tau_stats(self): self.print_start("Rho stats") for ver in self.versions: + base = self.basename(ver) rho_stat_handler, tau_stat_handler = get_rho_tau_w_cov( self.cc, ver, self.treecorr_config, out_dir, + base, method=self.cov_estimate_method, cov_rho=self.compute_cov_rho, + npatch=self.npatch, ) self.print_done("Rho stats finished") @@ -726,7 +744,9 @@ def calculate_ellipticity_dispersion(self): self._ellipticity_dispersion = ellipticity_dispersion def plot_rho_stats(self, abs=False): - filenames = [f"rho_stats_{ver}.fits" for ver in self.versions] + filenames = [ + f"rho_stats_{self.basename(ver)}.fits" for ver in self.versions + ] savefig = "rho_stats.png" self.rho_stat_handler.plot_rho_stats( @@ -746,7 +766,9 @@ def plot_rho_stats(self, abs=False): ) def plot_tau_stats(self, plot_tau_m=False): - filenames = [f"tau_stats_{ver}.fits" for ver in self.versions] + filenames = [ + f"tau_stats_{self.basename(ver)}.fits" for ver in self.versions + ] savefig = "tau_stats.png" self.tau_stat_handler.plot_tau_stats( @@ -826,9 +848,12 @@ def calculate_rho_tau_fits(self): self.cov_estimate_method, None ) + base = self.basename(ver) + flat_samples, result, q = get_samples( self.psf_fitter, ver, + base, cov_type=self.cov_estimate_method, apply_debias=npatch, sampler=self.rho_tau_method, @@ -838,7 +863,7 @@ def calculate_rho_tau_fits(self): self.rho_tau_fits["result_list"].append(result) self.rho_tau_fits["q_list"].append(q) - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] xi_psf_sys_samples = np.array([]).reshape(0, nbins) @@ -883,7 +908,7 @@ def plot_rho_tau_fits(self): self.colors, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") for i in range(100): self.psf_fitter.plot_xi_psf_sys( flat_sample[-i + 1], ver, color, alpha=0.1 @@ -933,7 +958,7 @@ def plot_rho_tau_fits(self): self.versions, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") for yscale in ("linear", "log"): out_path = os.path.abspath( f"{out_dir}/xi_psf_sys_terms_{yscale}_{ver}.png" @@ -2582,10 +2607,16 @@ def calculate_pseudo_cl_eb_cov(self): lmax = 2 * self.nside z, dndz = self.get_redshift(ver) + ell = np.arange(1, lmax + 1) pw = hp.pixwin(nside, lmax=lmax) + if pw.shape[0] != len(ell) + 1: + raise ValueError( + "Unexpected pixwin length for lmax=" + f"{lmax}: got {pw.shape[0]}, expected {len(ell)+1}" + ) + pw = pw[1:len(ell)+1] # Load redshift distribution and calculate theory C_ell - ell = np.arange(1, lmax + 1) fiducial_cl = ( get_theo_c_ell( ell=ell, diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 29ac84a..2647b37 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -1,5 +1,6 @@ import os import time +from pathlib import Path import matplotlib.scale as mscale import matplotlib.ticker as ticker @@ -69,6 +70,9 @@ def get_transform(self): ] +def _extract_xip(correlations): + """Return flattened array of xip values from a list of correlations.""" + return np.array([corr.xip for corr in correlations]).flatten() def get_params_rho_tau(cat, survey="other"): # Set parameters @@ -111,42 +115,59 @@ def get_params_rho_tau(cat, survey="other"): return params -def get_rho_tau_w_cov(config, version, treecorr_config, outdir, method, cov_rho=False): - """ - Method to compute the covariance matrices of rho and tau-statistics of a given list of versions in cosmo_val. - Also computes rho and tau-statistics. - - Parameters - ---------- - versions : list - List of versions to compute the covariance matrices for. - method : str - Method to compute the covariance matrices. Options are 'jk' or 'th'. - """ +def get_rho_tau_w_cov( + config, + version, + treecorr_config, + outdir, + base, + method, + cov_rho=False, + npatch=None, +): + """Compute rho/tau statistics and, if requested, their covariance.""" if method == "th": nbin_ang, nbin_rad = 100, 200 rho_stat_handler, tau_stat_handler = get_rho_tau( - config, version, treecorr_config, outdir, cov_rho=cov_rho + config, + version, + treecorr_config, + outdir, + base, + cov_rho=cov_rho, ) get_theory_cov( config, version, treecorr_config, outdir, + base, nbin_ang=nbin_ang, nbin_rad=nbin_rad, ) return rho_stat_handler, tau_stat_handler elif method == "jk": - return get_jackknife_cov(config, version, treecorr_config, outdir) + return get_jackknife_cov( + config, + version, + treecorr_config, + outdir, + base, + npatch=npatch, + ) elif method == "sim": - if os.path.exists(outdir + "/cov_tau_" + version + "_th.npy"): - print( - f"Covariance from simulation available at the following file: {outdir+'/cov_tau_'+version+'_th.npy'}" - ) - print(f"Computing rho and tau statistics for the version: {version}") + tau_cov_path = Path(outdir) / f"cov_tau_{base}_th.npy" + + if tau_cov_path.exists(): + print(f"Found existing covariance at {tau_cov_path}") + print(f"Computing rho/tau statistics for {version}") return get_rho_tau( - config, version, treecorr_config, outdir, cov_rho=cov_rho + config, + version, + treecorr_config, + outdir, + base, + cov_rho=cov_rho, ) else: raise ValueError( @@ -156,7 +177,15 @@ def get_rho_tau_w_cov(config, version, treecorr_config, outdir, method, cov_rho= raise ValueError("Method must be either 'jk' or 'th' or 'sim'.") -def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): +def get_rho_tau( + config, + version, + treecorr_config, + outdir, + base, + cov_rho=False, + npatch=None, +): """ Compute rho and tau statistics for a given version of the catalogue. @@ -167,7 +196,7 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): version : str Version of the catalogue to use. treecorr_config : dict - Configuraion for treecorr. + TreeCorr configuration (must include 'min_sep', 'max_sep', and 'nbins'). outdir : str Output directory. """ @@ -177,16 +206,15 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): print("Compute Rho and Tau statistics for the version: ", version) start_time = time.time() - out_base = f"rho_stats_{version}.fits" - out_path = f"{outdir}/{out_base}" + rho_path = Path(outdir) / f"rho_stats_{base}.fits" rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=True ) - if os.path.exists(out_path): - print(f"Skipping rho statistics computation, file {out_path} already exists.") - rho_stat_handler.load_rho_stats(out_base) + if rho_path.exists(): + print(f"Skipping rho statistics computation, file {rho_path} already exists.") + rho_stat_handler.load_rho_stats(rho_path.name) else: rho_stat_handler.catalogs.set_params(params, outdir) @@ -208,20 +236,19 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): ) if cov_rho: - if not os.path.exists(outdir + "/cov_rho_" + version + ".npy"): - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() + cov_rho_path = Path(outdir) / f"cov_rho_{base}_jk.npy" + if not cov_rho_path.exists(): rho_stat_handler.compute_rho_stats( version, - out_base, + rho_path.name, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) else: - rho_stat_handler.compute_rho_stats(version, out_base, var_method=None) + rho_stat_handler.compute_rho_stats(version, rho_path.name, var_method=None) - out_base = f"tau_stats_{version}.fits" - out_path = f"{outdir}/{out_base}" + tau_path = Path(outdir) / f"tau_stats_{base}.fits" tau_stat_handler = TauStat( catalogs=rho_stat_handler.catalogs, @@ -230,9 +257,9 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): verbose=True, ) - if os.path.exists(out_path): - print(f"Skipping tau statistics computation, file {out_path} already exists.") - tau_stat_handler.load_tau_stats(out_base) + if tau_path.exists(): + print(f"Skipping tau statistics computation, file {tau_path} already exists.") + tau_stat_handler.load_tau_stats(tau_path.name) else: tau_stat_handler.catalogs.set_params(params, outdir) @@ -266,14 +293,20 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): ) # function to extract the tau_+ - tau_stat_handler.compute_tau_stats(version, out_base, var_method=None) + tau_stat_handler.compute_tau_stats(version, tau_path.name, var_method=None) print(f"Time to compute rho and tau statistics: {time.time() - start_time:.2f} s") return rho_stat_handler, tau_stat_handler def get_theory_cov( - config, version, treecorr_config, outdir, nbin_ang=100, nbin_rad=100 + config, + version, + treecorr_config, + outdir, + base, + nbin_ang=100, + nbin_rad=100, ): """ Compute an analytical estimate of the covariance matrix of rho and tau-statistics. @@ -290,15 +323,17 @@ def get_theory_cov( path_psf = info["psf"]["path"] hdu_psf = info["psf"]["hdu"] - print("Computing the covariance matrix for the version: ", version) - start_time = time.time() + target_cov = Path(outdir) / f"cov_tau_{base}_th.npy" - if os.path.exists(outdir + "/cov_tau_" + version + "_th.npy"): + if target_cov.exists(): print( - f"Skipping covariance computation, file {outdir+'/cov_tau_'+version+'_th.npy'} already exists." + f"Skipping covariance computation, file {target_cov} already exists." ) return + print("Computing the covariance matrix for the version: ", version) + start_time = time.time() + cov_tau_th = CovTauTh( path_gal=path_gal, path_psf=path_psf, @@ -310,28 +345,39 @@ def get_theory_cov( params=params, ) - print( - "--- Computation of the rho and tau statistics for the covariance %s seconds ---" - % (time.time() - start_time) - ) + elapsed = time.time() - start_time + print(f"--- Rho/tau statistics for covariance computed in {elapsed:.2f}s ---") cov = cov_tau_th.build_cov(nbin_ang=nbin_ang, nbin_rad=nbin_rad) - print("--- Covariance computation %s seconds ---" % (time.time() - start_time)) - np.save(outdir + "/cov_tau_" + version + "_th.npy", cov) + print(f"--- Covariance matrix assembled in {time.time() - start_time:.2f}s ---") + target_cov.parent.mkdir(parents=True, exist_ok=True) + np.save(target_cov, cov) print("Saved covariance matrix of version: ", version) del cov_tau_th return -def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): +def get_jackknife_cov( + config, + version, + treecorr_config, + outdir, + base, + npatch, + ncov=100, +): """ Compute the covariance matrix of rho and tau-statistics using the jackknife method. Also compute rho and tau-statistics. """ - if os.path.exists(outdir + "/cov_tau_" + version + "_jk.npy"): + rho_filename = f"rho_stats_{base}.fits" + tau_filename = f"tau_stats_{base}.fits" + tau_cov_path = Path(outdir) / f"cov_tau_{base}_jk.npy" + + if tau_cov_path.exists(): print( - f"Skipping covariance computation, file {outdir+'/cov_tau_'+version+'_jk.npy'} already exists." + f"Skipping covariance computation, file {tau_cov_path} already exists." ) rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=False @@ -344,19 +390,22 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): verbose=True, ) + rho_path = Path(outdir) / rho_filename + tau_path = Path(outdir) / tau_filename + if rho_path.exists(): + rho_stat_handler.load_rho_stats(rho_path.name) + if tau_path.exists(): + tau_stat_handler.load_tau_stats(tau_path.name) return rho_stat_handler, tau_stat_handler - + params = get_params_rho_tau(config[version], survey=version) rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=False ) - out_base = f"rho_stats_{version}.fits" - rho_stat_handler.catalogs.set_params(params, outdir) - mask = version != "DES" square_size = params["square_size"] tau_stat_handler = TauStat( @@ -366,25 +415,18 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): verbose=True, ) - out_base = f"tau_stats_{version}.fits" - tau_stat_handler.catalogs.set_params(params, outdir) for i in range(ncov): - if not ( - os.path.exists(outdir + "/cov_tau_" + version + str(i) + ".npy") - and os.path.exists(outdir + "/cov_rho_" + version + str(i) + ".npy") - ): - + tau_chunk = outdir + f"/cov_tau_{version}{i}.npy" + rho_chunk = outdir + f"/cov_rho_{version}{i}.npy" + if not (os.path.exists(tau_chunk) and os.path.exists(rho_chunk)): print( - f"Computing rho-statistics of version {version} for jackknife patch {i+1}/{ncov}" + f"Computing rho-statistics for {version} (patch {i+1}/{ncov})" ) - if ( - f"psf_{version}{i}" not in rho_stat_handler.catalogs.catalogs_dict.keys() - ): - + if f"psf_{version}{i}" not in rho_stat_handler.catalogs.catalogs_dict: # Build catalogues rho_stat_handler.build_cat_to_compute_rho( config[version]["psf"]["path"], @@ -394,7 +436,9 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): hdu=config[version]["psf"]["hdu"], ) - tau_stat_handler.catalogs.catalogs_dict = rho_stat_handler.catalogs.catalogs_dict + tau_stat_handler.catalogs.catalogs_dict = ( + rho_stat_handler.catalogs.catalogs_dict + ) # Build the catalog of galaxies. PSF was computed above tau_stat_handler.build_cat_to_compute_tau( @@ -411,45 +455,48 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): ) npatch = rho_stat_handler.catalogs._params["patch_number"] - field = rho_stat_handler.catalogs.catalogs_dict[f"psf_{version}{i}"].getNField(max_top=int.bit_length(npatch)-1, coords='spherical') + field = rho_stat_handler.catalogs.catalogs_dict[ + f"psf_{version}{i}" + ].getNField(max_top=int.bit_length(npatch) - 1, coords="spherical") patch, centers = field.run_kmeans(npatch) #Update the patch centers of the catalogs for key, cat in rho_stat_handler.catalogs.catalogs_dict.items(): cat._centers = centers - field = cat.getNField(max_top=int.bit_length(npatch)-1, coords='spherical') + field = cat.getNField( + max_top=int.bit_length(npatch) - 1, coords="spherical" + ) cat._patch = field.kmeans_assign_patches(centers) - - - out_base = f"rho_stats_{version}.fits" # Compute and save rho stats - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() rho_stat_handler.compute_rho_stats( version + str(i), - out_base, + rho_filename, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) - out_base = f"tau_stats_{version}.fits" - # function to extract the tau_+ - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() tau_stat_handler.compute_tau_stats( version + str(i), - out_base, + tau_filename, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) #Update the keys in the dictionaries - rho_stat_handler.catalogs.catalogs_dict[f"psf_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_{version}{i}") - rho_stat_handler.catalogs.catalogs_dict[f"psf_error_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_error_{version}{i}") - rho_stat_handler.catalogs.catalogs_dict[f"psf_size_error_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_size_error_{version}{i}") - tau_stat_handler.catalogs.catalogs_dict[f"gal_{version}{i+1}"] = tau_stat_handler.catalogs.catalogs_dict.pop(f"gal_{version}{i}") + rho_dict = rho_stat_handler.catalogs.catalogs_dict + tau_dict = tau_stat_handler.catalogs.catalogs_dict + rho_dict[f"psf_{version}{i+1}"] = rho_dict.pop(f"psf_{version}{i}") + rho_dict[f"psf_error_{version}{i+1}"] = rho_dict.pop( + f"psf_error_{version}{i}" + ) + rho_dict[f"psf_size_error_{version}{i+1}"] = rho_dict.pop( + f"psf_size_error_{version}{i}" + ) + tau_dict[f"gal_{version}{i+1}"] = tau_dict.pop(f"gal_{version}{i}") cov_tau_loc = np.zeros_like(np.load(outdir + f"/cov_tau_{version}0.npy")) cov_rho_loc = np.zeros_like(np.load(outdir + f"/cov_rho_{version}0.npy")) @@ -462,122 +509,162 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): cov_tau = cov_tau_loc / ncov cov_rho = cov_rho_loc / ncov - np.save(outdir + "/cov_tau_" + version + "_jk.npy", cov_tau) - np.save(outdir + "/cov_rho_" + version + ".npy", cov_rho) + tau_cov_path.parent.mkdir(parents=True, exist_ok=True) + np.save(tau_cov_path, cov_tau) + + cov_rho_path = Path(outdir) / f"cov_rho_{base}_jk.npy" + cov_rho_path.parent.mkdir(parents=True, exist_ok=True) + np.save(cov_rho_path, cov_rho) return rho_stat_handler, tau_stat_handler -def get_samples(psf_fitter, version, cov_type="jk", apply_debias=None, sampler="emcee"): - """ - Samples (alpha, beta, eta) using the sampler 'emcee' or 'lsq' +def get_samples( + psf_fitter, + version, + base, + cov_type="jk", + apply_debias=None, + sampler="emcee", +): + """Return (alpha, beta, eta) samples using ``emcee`` or least squares. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance that provides ``load_*`` helpers. version : str - Version of the catalogue to use. - cov_type : str - Type of covariance matrix to use. Options are 'jk' or 'th' or 'sim'. - apply_debias : int - If not None, apply debiasing to the least square method. - sampler : str - Sampler to use. Options are 'emcee' or 'lsq'. (Default: 'emcee') + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename (e.g. ``SP_v1.4_minsep=…``) used for filenames. + cov_type : str, optional + Covariance label (``'jk'``, ``'th'``, or ``'sim'``). Defaults to ``'jk'``. + apply_debias : int or None, optional + Jackknife patch count used to debias samples. Disabled when ``None``. + sampler : str, optional + ``'emcee'`` for MCMC sampling, ``'lsq'`` for least squares + (default ``'emcee'``). """ if sampler == "emcee": return get_samples_emcee( - psf_fitter, version, cov_type=cov_type, apply_debias=apply_debias + psf_fitter, + version, + base, + cov_type=cov_type, + apply_debias=apply_debias, ) elif sampler == "lsq": return get_samples_lsq( - psf_fitter, version, cov_type=cov_type, apply_debias=apply_debias + psf_fitter, + version, + base, + cov_type=cov_type, + apply_debias=apply_debias, ) else: raise ValueError("Sampler must be either 'emcee' or 'lsq'.") def get_samples_emcee( - psf_fitter, version, nwalkers=124, nsamples=10000, cov_type="jk", apply_debias=None + psf_fitter, + version, + base, + nwalkers=124, + nsamples=10000, + cov_type="jk", + apply_debias=None, ): - """ - Samples (alpha, beta, eta) using the covariance of the tau statistics and emcee. + """Draw (alpha, beta, eta) samples using ``emcee`` and the tau covariance. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance managing rho/tau statistics and covariances. version : str - Version of the catalogue to use. - nwalkers : int - Number of walkers to use in the MCMC. (Default: 124) - nsamples : int - Number of samples to draw from the MCMC. (Default 10000) - cov_type : str - Type of covariance matrix to use. Options are 'jk' or 'th' or 'sim'. (Default: 'jk') + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename for locating statistics/covariance files. + nwalkers : int, optional + Number of walkers for the MCMC run (default ``124``). + nsamples : int, optional + Number of samples drawn per walker (default ``10000``). + cov_type : str, optional + Covariance label (``'jk'``/``'th'``/``'sim'``). Defaults to ``'jk'``. + apply_debias : int or None, optional + Jackknife patch count applied during debiasing. Disabled when ``None``. """ # Load rho and tau stats - psf_fitter.load_rho_stat("rho_stats_" + version + ".fits") - psf_fitter.load_tau_stat("tau_stats_" + version + ".fits") + psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") + psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") # Check if the path exists sample_file_path = psf_fitter.get_sample_file_path(version) if os.path.exists(sample_file_path): - print(f"Skipping sample computation, file {sample_file_path} already exists.") + print(f"Skipping sampling; {sample_file_path} exists.") flat_samples = psf_fitter.load_samples(version) mcmc_result, q = psf_fitter.get_mcmc_from_samples(version) print(mcmc_result) # Or run MCMC else: print("MCMC sampling") - psf_fitter.load_covariance("cov_tau_" + version + "_" + cov_type + ".npy") + cov_filename = f"cov_tau_{base}_{cov_type}.npy" + psf_fitter.load_covariance(cov_filename, cov_type="tau") - npatch = apply_debias if (apply_debias is not None) else None + debias_npatch = apply_debias if (apply_debias is not None) else None flat_samples, mcmc_result, q = psf_fitter.run_chain( nwalkers=nwalkers, nsamples=nsamples, - npatch=npatch, - apply_debias=npatch is not None, + npatch=debias_npatch, + apply_debias=debias_npatch is not None, savefig="mcmc_samples_" + version + ".png", ) psf_fitter.save_samples(flat_samples, version) return flat_samples, mcmc_result, q -def get_samples_lsq(psf_fitter, version, apply_debias=None, cov_type="jk"): - """ - Samples (alpha, beta, eta) using the covariance of the tau statistics and least square method. +def get_samples_lsq( + psf_fitter, + version, + base, + apply_debias=None, + cov_type="jk", +): + """Compute least-squares samples of (alpha, beta, eta) using the tau covariance. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance managing rho/tau statistics and covariances. version : str - Version of the catalogue to use. - apply_debias : int - If not None, apply debiasing to the least square method. (Default: None) + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename for locating statistics/covariance files. + apply_debias : int or None, optional + Jackknife patch count applied during debiasing. Disabled when ``None``. + cov_type : str, optional + Covariance label (defaults to ``'jk'``). """ # Load rho and tau stats - psf_fitter.load_rho_stat("rho_stats_" + version + ".fits") - psf_fitter.load_tau_stat("tau_stats_" + version + ".fits") + psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") + psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") # Check if the path exists sample_file_path = psf_fitter.get_sample_path(version) if os.path.exists(sample_file_path): - print(f"Skipping sample computation, file {sample_file_path} already exists.") + print(f"Skipping sampling; {sample_file_path} exists.") flat_samples = psf_fitter.load_samples(version) mcmc_result, q = psf_fitter.get_mcmc_from_samples(flat_samples) print(mcmc_result) # Or run MCMC else: print("Least square sampling") - psf_fitter.load_covariance( - "cov_tau_" + version + "_" + cov_type + ".npy", cov_type="tau" - ) - psf_fitter.load_covariance("cov_rho_" + version + ".npy", cov_type="rho") - npatch = apply_debias if (apply_debias is not None) else None + tau_covariance = f"cov_tau_{base}_{cov_type}.npy" + rho_covariance = f"cov_rho_{base}_jk.npy" + psf_fitter.load_covariance(tau_covariance, cov_type="tau") + psf_fitter.load_covariance(rho_covariance, cov_type="rho") + debias_npatch = apply_debias if (apply_debias is not None) else None flat_samples, mcmc_result, q = psf_fitter.get_least_squares_params_samples( - npatch=npatch, apply_debias=(npatch is not None) + npatch=debias_npatch, apply_debias=(debias_npatch is not None) ) psf_fitter.save_samples(flat_samples, version) return flat_samples, mcmc_result, q From a1bd87224e5505f0e6a3d4fdb009d5c68ca90ffd Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 27 Oct 2025 23:38:33 +0100 Subject: [PATCH 18/34] merge pipeline.sh and cosmosis_fitting, make work with mocks --- cosmo_inference/scripts/cosmosis_fitting.py | 813 ++++++++++++-------- 1 file changed, 478 insertions(+), 335 deletions(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 7f4d918..6c2fc8f 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -1,142 +1,215 @@ #!/usr/bin/env python # coding: utf-8 +import argparse import os +import re import sys -from pathlib import Path -import matplotlib.pylab as plt import numpy as np from astropy.io import fits +# ============================================================================ +# SECTION 1: FITS FORMAT CONVERTERS (data → FITS HDU) +# ============================================================================ -# transforms treecorr fits file of correlation functions into CosmoSIS-friendly 2pt FITS extension to be read by 2pt_likelihood -def treecorr_to_fits(filename1, filename2): +def _create_2pt_hdu(values, theta, name, quant1, quant2): + """Create standardized 2-point correlation FITS HDU.""" + nbins = len(values) + lst = np.arange(1, nbins + 1) + + col1 = fits.Column(name="BIN1", format="K", array=np.ones(nbins)) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(nbins)) + col3 = fits.Column(name="ANGBIN", format="K", array=lst) + col4 = fits.Column(name="VALUE", format="D", array=values) + col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=theta) + + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + hdu = fits.BinTableHDU.from_columns(coldefs, name=name) + + hdu_dict = { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + } + for key, value in hdu_dict.items(): + hdu.header[key] = value + + return hdu + + +def nz_to_fits(filename): + """Convert n(z) text file to FITS format.""" + line = np.loadtxt(filename, max_rows=1) + nbins = len(line) - 1 + + z_low = np.loadtxt(filename, usecols=0) + nstep = z_low[1] - z_low[0] + + z_mid = z_low + nstep / 2 + z_high = np.append(z_low[1:], z_low[-1] + nstep) + + col1 = fits.Column(name="Z_LOW", format="D", array=z_low) + col2 = fits.Column(name="Z_MID", format="D", array=z_mid) + col3 = fits.Column(name="Z_HIGH", format="D", array=z_high) + cols = [col1, col2, col3] + + for i in range(nbins): + bin_col = np.loadtxt(filename, usecols=i + 1) + hdu_col = fits.Column(name="BIN%d" % (i + 1), format="D", array=bin_col) + cols.append(hdu_col) + + coldefs = fits.ColDefs(cols) + nz_hdu = fits.BinTableHDU.from_columns(coldefs, name="NZDATA") + + nz_lens_dict = { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": nbins, + "NZ": len(z_low), + } + + for key, value in nz_lens_dict.items(): + nz_hdu.header[key] = value + + return nz_hdu + + +def treecorr_to_fits(filename1, filename2): + """Load xi+ and xi- from separate TreeCorr FITS files.""" xiplus_hdu = fits.open(filename1) ximinus_hdu = fits.open(filename2) - return xiplus_hdu[1], ximinus_hdu[1] -def tau_to_fits(filename, theta=None): +def parse_combined_xi_fits(filepath): + """Parse combined FITS file with xip/xim columns (mock format).""" + with fits.open(filepath) as hdul: + xi_data = hdul[1].data + xip_vals = xi_data["xip"] + xim_vals = xi_data["xim"] + xi_theta = xi_data["ANG"] if "ANG" in xi_data.names else xi_data["meanr"] + + xip_hdu = _create_2pt_hdu(xip_vals, xi_theta, "XI_PLUS", "P+P", "P+P") + xim_hdu = _create_2pt_hdu(xim_vals, xi_theta, "XI_MINUS", "P+P", "P+P") + + return xip_hdu, xim_hdu + + +def load_glass_cl(cl_file): """ - Convert tau statistics to CosmoSIS FITS format. - - Parameters: - filename : str - Path to tau statistics FITS file - theta : array-like, optional - Angular separation values to use. If provided, overrides the theta values - from the tau statistics file. This is useful for forcing consistency with - xi correlation function angular separations. + Load GLASS mock C_ell from .npy file. + Expected shape: (5, 32) where row 0=ell, row 1=EE, row 4=BB """ - tau_stats = fits.getdata(filename) + cl_block = np.load(cl_file) + if cl_block.shape[0] < 5: + raise ValueError(f"Unexpected C_ell array shape {cl_block.shape} for {cl_file}") - # Use provided theta if given, otherwise use tau's original theta values - if theta is not None: - ang = theta - print(f"Using provided theta values for tau statistics (forcing consistency)") - else: - ang = tau_stats["theta"] - print(f"Using original tau theta values") - - nbins = len(ang) + ell = np.asarray(cl_block[0], dtype=np.float64) + cl_ee = np.asarray(cl_block[1], dtype=np.float64) + cl_bb = np.asarray(cl_block[4], dtype=np.float64) + + return ell, cl_ee, cl_bb + + +def glass_cl_to_fits(ell, cl_ee, cl_bb): + """Convert GLASS C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" + nbins = len(ell) lst = np.arange(1, nbins + 1) - # Create fits HDU for tau_0_+ - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) + col1 = fits.Column(name="BIN1", format="K", array=np.ones(nbins)) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(nbins)) col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=tau_stats["tau_0_p"]) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=ang) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - tau_0_p_hdu = fits.BinTableHDU.from_columns(coldefs, name="TAU_0_PLUS") + col5 = fits.Column(name="ANG", format="D", array=ell) - # Create fits HDU for tau_2_+ - col4 = fits.Column(name="VALUE", format="D", array=tau_stats["tau_2_p"]) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - tau_2_p_hdu = fits.BinTableHDU.from_columns(coldefs, name="TAU_2_PLUS") + col4_ee = fits.Column(name="VALUE", format="D", array=cl_ee) + coldefs_ee = fits.ColDefs([col1, col2, col3, col4_ee, col5]) + cl_ee_hdu = fits.BinTableHDU.from_columns(coldefs_ee, name="CELL_EE") - # Append tau_0_p/tau_2_p header info - tau_0_p_dict = { + cl_ee_dict = { "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "P+R", + "QUANT1": "GEF", + "QUANT2": "GEF", "KERNEL_1": "NZ_SOURCE", "KERNEL_2": "NZ_SOURCE", "WINDOWS": "SAMPLE", } + for key, value in cl_ee_dict.items(): + cl_ee_hdu.header[key] = value - for key in tau_0_p_dict: - tau_0_p_hdu.header[key] = tau_0_p_dict[key] + col4_bb = fits.Column(name="VALUE", format="D", array=cl_bb) + coldefs_bb = fits.ColDefs([col1, col2, col3, col4_bb, col5]) + cl_bb_hdu = fits.BinTableHDU.from_columns(coldefs_bb, name="CELL_BB") - tau_2_p_dict = { + cl_bb_dict = { "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "SR+R", + "QUANT1": "GBF", + "QUANT2": "GBF", "KERNEL_1": "NZ_SOURCE", "KERNEL_2": "NZ_SOURCE", "WINDOWS": "SAMPLE", } + for key, value in cl_bb_dict.items(): + cl_bb_hdu.header[key] = value - for key in tau_2_p_dict: - tau_2_p_hdu.header[key] = tau_2_p_dict[key] + return cl_ee_hdu, cl_bb_hdu - return tau_0_p_hdu, tau_2_p_hdu -def pseudo_cl_to_fits(filename): - pseudo_cl = fits.getdata(filename) - cl_ee = pseudo_cl["EE"] - ell = pseudo_cl["ELL"] +def tau_to_fits(filename, theta=None): + """ + Convert tau statistics to FITS format. + If theta provided, override original values for consistency with xi. + """ + tau_stats = fits.getdata(filename) - nbins = len(ell) - lst = np.arange(1, nbins+1) - - #Create fits HDU for tau_0_+ - col1 = fits.Column(name ='BIN1', format ='K', array = np.ones(len(lst))) - col2 = fits.Column(name ='BIN2', format ='K', array = np.ones(len(lst))) - col3 = fits.Column(name ='ANGBIN', format ='K', array = lst) - col4 = fits.Column(name = 'VALUE', format = 'D', array = cl_ee) - col5 = fits.Column(name = 'ANG', format = 'D', array = ell) - coldefs = fits.ColDefs([col1,col2,col3,col4,col5]) - pseudo_cl_hdu = fits.BinTableHDU.from_columns(coldefs,name ='CELL_EE') - - pseudo_cl_dict = { - '2PTDATA': 'T', - 'QUANT1' : 'GEF', - 'QUANT2' : 'GEF', - 'KERNEL_1': 'NZ_SOURCE', - 'KERNEL_2': 'NZ_SOURCE', - 'WINDOWS': 'SAMPLE' - } + if theta is not None: + ang = theta + print("Using provided theta values for tau statistics (forcing consistency)") + else: + ang = tau_stats["theta"] + print("Using original tau theta values") - for key in pseudo_cl_dict: - pseudo_cl_hdu.header[key] = pseudo_cl_dict[key] + tau_0_p_hdu = _create_2pt_hdu(tau_stats["tau_0_p"], ang, "TAU_0_PLUS", "G+R", "P+R") + tau_2_p_hdu = _create_2pt_hdu( + tau_stats["tau_2_p"], ang, "TAU_2_PLUS", "G+R", "SR+R" + ) - return pseudo_cl_hdu + return tau_0_p_hdu, tau_2_p_hdu -def cov_pseudo_cl_to_fits(filename): - cov_pseudo_cl = fits.open(filename) - cov_ee = cov_pseudo_cl["COVAR_FULL"].data - cov_hdu = fits.ImageHDU(cov_ee) - cov_dict = { - 'COVDATA': 'True', - 'EXTNAME': 'COVMAT', - 'NAME_0': 'CELL_EE', - 'STRT_0': 0, - } +def rho_to_fits(filename, theta=None): + """ + Convert rho statistics to FITS format. + If theta provided, override original values for consistency with xi. + """ + rho_stat_hdul = fits.open(filename) + rho_stat_hdu = rho_stat_hdul[1].copy() + rho_stat_hdu.name = "RHO_STATS" - for key in cov_dict: - cov_hdu.header[key] = cov_dict[key] + if theta is not None: + print( + "Forcing rho statistics to use provided theta values (forcing consistency)" + ) + rho_stat_hdu.data = rho_stat_hdu.data.copy() + rho_stat_hdu.data["theta"] = theta + else: + print("Using original rho theta values") + + rho_stat_hdul.close() + return rho_stat_hdu - return cov_hdu -#transforms text file of CosmoCov data into covmat HDU extension def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): + """ + Convert CosmoCov covariance matrix to FITS format. - # read in cov txt data from CosmoCov + If tau covariance provided, block with xi covariance. + """ covmat_xi = np.loadtxt(filename_cov_xi) + if filename_cov_tau is not None: covmat_tau = np.load(filename_cov_tau) nbins = int(len(covmat_tau) / 3) @@ -147,282 +220,352 @@ def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): [np.zeros((len(covmat_tau), len(covmat_xi))), covmat_tau], ] ) + else: + covmat = covmat_xi if len(covmat) != len(covmat[0]): - print("Error: covmat not square!") - exit() + raise RuntimeError("Covariance matrix is not square") - else: - # create covmat ImageHDU - cov_hdu = fits.ImageHDU(covmat) - - # create header - if filename_cov_tau is None: - cov_dict = { - "COVDATA": "True", - "EXTNAME": "COVMAT", - "NAME_0": "XI_PLUS", - "STRT_0": 0, - "NAME_1": "XI_MINUS", - "STRT_1": int(len(covmat) / 2), - } - else: - cov_dict = { - "COVDATA": "True", - "EXTNAME": "COVMAT", - "NAME_0": "XI_PLUS", - "STRT_0": 0, - "NAME_1": "XI_MINUS", - "STRT_1": int(len(covmat_xi) / 2), - "NAME_2": "TAU_0_PLUS", - "STRT_2": len(covmat_xi), - "NAME_3": "TAU_2_PLUS", - "STRT_3": len(covmat_xi) + int(len(covmat_tau) / 2), - } - for key in cov_dict: - cov_hdu.header[key] = cov_dict[key] + cov_hdu = fits.ImageHDU(covmat) + + cov_dict = { + "COVDATA": "True", + "EXTNAME": "COVMAT", + "NAME_0": "XI_PLUS", + "STRT_0": 0, + "NAME_1": "XI_MINUS", + "STRT_1": int(len(covmat) / 2), + } + + filename_cov_tau and cov_dict.update({ + "NAME_2": "TAU_0_PLUS", + "STRT_2": len(covmat_xi), + "NAME_3": "TAU_2_PLUS", + "STRT_3": len(covmat_xi) + int(len(covmat_tau) / 2), + }) + + for key, value in cov_dict.items(): + cov_hdu.header[key] = value return cov_hdu -# transforms nz data (that was used in CosmoCov format) into nzdat HDU extension -def nz_to_fits(filename): +# ============================================================================ +# SECTION 2: VALIDATION & CONSISTENCY CHECKS +# ============================================================================ - line = np.loadtxt(filename, max_rows=1) - nbins = len(line) - 1 - z_low = np.loadtxt(filename, usecols=0) +def check_meanr_consistency(xi_theta, tau_theta, rho_theta, threshold=1.0): + """Check and report theta consistency across xi, tau, and rho statistics.""" + print("=" * 60) + print("MEANR CONSISTENCY CHECK") + print("=" * 60) - nstep = z_low[1] - z_low[0] + tau_diff = np.abs((tau_theta - xi_theta) / xi_theta) * 100 + rho_diff = np.abs((rho_theta - xi_theta) / xi_theta) * 100 - z_mid = z_low + nstep / 2 - z_high = np.append(z_low[1:], z_low[-1] + nstep) + print(f"Xi theta range: {xi_theta.min():.6f} - {xi_theta.max():.6f} arcmin") + print(f"Tau theta range: {tau_theta.min():.6f} - {tau_theta.max():.6f} arcmin") + print(f"Rho theta range: {rho_theta.min():.6f} - {rho_theta.max():.6f} arcmin") + print() + print(f"Max tau-xi relative difference: {tau_diff.max():.3f}%") + print(f"Mean tau-xi relative difference: {tau_diff.mean():.3f}%") + print(f"Max rho-xi relative difference: {rho_diff.max():.3f}%") + print(f"Mean rho-xi relative difference: {rho_diff.mean():.3f}%") + + if tau_diff.max() > threshold or rho_diff.max() > threshold: + print(f"\nWARNING: Meanr differences exceed {threshold}% threshold") + if tau_diff.max() > threshold: + print(f" Tau-xi: max {tau_diff.max():.3f}%") + if rho_diff.max() > threshold: + print(f" Rho-xi: max {rho_diff.max():.3f}%") + else: + print(f"✓ All differences below {threshold}% threshold") - # create hdu for histogram bin - col1 = fits.Column(name="Z_LOW", format="D", array=z_low) - col2 = fits.Column(name="Z_MID", format="D", array=z_mid) - col3 = fits.Column(name="Z_HIGH", format="D", array=z_high) - cols = [col1, col2, col3] + print("=" * 60) + print() - for i in range(nbins): - bin_col = np.loadtxt(filename, usecols=i + 1) - hdu_col = fits.Column(name="BIN%d" % (i + 1), format="D", array=bin_col) - cols.append(hdu_col) - coldefs = fits.ColDefs(cols) - nz_hdu = fits.BinTableHDU.from_columns(coldefs, name="NZDATA") +# ============================================================================ +# SECTION 3: COSMOSIS CONFIGURATION GENERATION +# ============================================================================ + + +def _generate_ini_file( + args, + template_base, + priors_file, + values_file, + suffix="", + is_harmonic=False, +): + """Generate a CosmoSIS INI configuration file from template with modifications.""" + template_path = f"cosmosis_config/{template_base}" + output_path = f"cosmosis_config/cosmosis_pipeline_{args.cosmosis_root}{suffix}.ini" + + with open(template_path, "r") as f: + config_content = f.read() + + modifications = [] + + relative_fits_file = f"data/{args.cosmosis_root}/cosmosis_{args.cosmosis_root}.fits" + default_section = ( + f"[DEFAULT]\nSCRATCH = {args.data_dir}\nFITS_FILE = {relative_fits_file}" + ) + modifications.append((r"^\[DEFAULT\]", default_section)) + + output_section = ( + f"[output]\nfilename = %(SCRATCH)s/{args.cosmosis_root}/samples_" + f"{args.cosmosis_root}{suffix}.txt" + ) + modifications.append((r"^\[output\]", output_section)) + + pipeline_section = ( + f"[pipeline]\nvalues = cosmosis_config/{values_file}\npriors = " + f"cosmosis_config/{priors_file}" + ) + modifications.append((r"^\[pipeline\]", pipeline_section)) + + if not is_harmonic: + if args.use_rho_tau: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py" + "\ndata_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS\nadd_xi_sys=T" + ) + else: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" + "\ndata_sets=XI_PLUS XI_MINUS" + ) + else: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" + "\ndata_sets=CELL_EE CELL_BB" + ) - # create n(z) header - nz_lens_dict = { - "NZDATA": "T ", - "EXTNAME": "NZ_SOURCE", - "NBIN": nbins, - "NZ": len(z_low), - } + modifications.append((r"^\[2pt_like\]", like_section)) - for key in nz_lens_dict: - nz_hdu.header[key] = nz_lens_dict[key] + poly_section = f"[polychord]\npolychord_outfile_root = {args.cosmosis_root}{suffix}" + modifications.append((r"^\[polychord\]", poly_section)) - return nz_hdu + test_section = ( + f"[test]\nsave_dir = %(SCRATCH)s/best_fit/{args.cosmosis_root}{suffix}" + ) + modifications.append((r"^\[test\]", test_section)) + for pattern, replacement in modifications: + config_content = re.sub( + pattern, replacement, config_content, flags=re.MULTILINE + ) -def rho_to_fits(filename, theta=None): - """ - Convert rho statistics to CosmoSIS FITS format. - - Parameters: - filename : str - Path to rho statistics FITS file - theta : array-like, optional - Angular separation values to use. If provided, replaces the theta values - in the rho statistics file. This is useful for forcing consistency with - xi correlation function angular separations. - """ - rho_stat_hdul = fits.open(filename) - rho_stat_hdu = rho_stat_hdul[1].copy() # Create a copy to avoid modifying the original - rho_stat_hdu.name = "RHO_STATS" - - # Force rho to use provided theta if given - if theta is not None: - print(f"Forcing rho statistics to use provided theta values (forcing consistency)") - # Update the theta column in the data - rho_stat_hdu.data = rho_stat_hdu.data.copy() # Make data writable - rho_stat_hdu.data['theta'] = theta - else: - print(f"Using original rho theta values") - - rho_stat_hdul.close() # Close the original file - return rho_stat_hdu - -if __name__ == "__main__": - -#combines all the data: 2pt correlation functions from treecorr, covmat from CosmoCov (must already be combined into 1 txt file), nz txt data -#into 1 fits file to be read by CosmoSIS 2pt-likelihood function -#give file path of each of the 3 components as input, also file path of desired output FITS file -#outputs nothing, but writes a new FITS file with appropriate extensions + with open(output_path, "w") as f: + f.write(config_content) - # Print all input arguments - print("=" * 60) - print("COSMOSIS_FITTING.PY - INPUT ARGUMENTS") - print("=" * 60) - print(f"Script name: {sys.argv[0]}") - print(f"Total arguments: {len(sys.argv)}") - - arg_names = [ - "script_name", - "cosmosis_root", - "output_folder", - "nz_file", - "use_pseudo_cell", - "out_file", - ] - - for i, arg in enumerate(sys.argv): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {arg}") - print("=" * 60) - print() + config_type = "harmonic-space" if is_harmonic else "real-space" + print(f"Prepared CosmoSIS {config_type} configuration file in {output_path}") + print(f"You can now run inference with the command: cosmosis {output_path}") - cosmosis_root = sys.argv[1] - output_folder = sys.argv[2] - nz_file = sys.argv[3] - out_file = sys.argv[5] +def generate_cosmosis_config(args): + """Generate CosmoSIS INI files (real-space and optional harmonic-space).""" + if args.use_rho_tau: + template_base_realspace = "cosmosis_pipeline_A_psf.ini" + priors_file = "priors_psf.ini" + values_file = "values_psf.ini" + else: + template_base_realspace = "cosmosis_pipeline_A_ia.ini" + priors_file = "priors.ini" + values_file = "values_ia.ini" + + os.makedirs("cosmosis_config", exist_ok=True) + + _generate_ini_file( + args, + template_base_realspace, + priors_file, + values_file, + suffix="", + is_harmonic=False, + ) + + if args.cl_file: + template_base_harmonic = "cosmosis_pipeline_A_ia_cell.ini" + _generate_ini_file( + args, + template_base_harmonic, + priors_file, + values_file, + suffix="_cell", + is_harmonic=True, + ) - use_pseudo_cell = sys.argv[4] - use_pseudo_cell = True if use_pseudo_cell == 'y' else False - if use_pseudo_cell: - arg_names += ["cl_root", "gaussian_part"] +# ============================================================================ +# SECTION 4: MAIN WORKFLOW +# ============================================================================ + + +def parse_args(): + """Parse command-line arguments for unified data/mock interface.""" + parser = argparse.ArgumentParser( + description="Prepare CosmoSIS inference FITS files from real or mock data. " + "Supports multiple xi input formats (separate files, combined FITS)." + ) + + parser.add_argument( + "--cosmosis-root", type=str, required=True, help="CosmoSIS root identifier" + ) + parser.add_argument( + "--data-dir", type=str, required=True, help="Output MCMC chain directory" + ) + parser.add_argument("--nz-file", type=str, required=True, help="Path to n(z) file") + parser.add_argument( + "--out-file", type=str, required=True, help="Path to output FITS file" + ) + parser.add_argument( + "--xi", + nargs="+", + required=True, + help="Xi files: 1 (mock FITS) or 2 (data: xi_plus.fits, xi_minus.fits)", + ) + parser.add_argument( + "--cov-xi", type=str, required=True, help="Xi covariance matrix file" + ) + + parser.add_argument( + "--use-rho-tau", + action="store_true", + help="Include PSF systematics (requires --rho-stats, --tau-stats, --cov-tau)", + ) + parser.add_argument( + "--rho-stats", + type=str, + required=False, + help="Path to rho statistics FITS file (required if --use-rho-tau)", + ) + parser.add_argument( + "--tau-stats", + type=str, + required=False, + help="Path to tau statistics FITS file (required if --use-rho-tau)", + ) + parser.add_argument( + "--cov-tau", + type=str, + required=False, + help="Path to tau covariance matrix (required if --use-rho-tau)", + ) + parser.add_argument( + "--cl-file", + type=str, + required=False, + help="Path to C_ell data file (.npy, optional for data and mock)", + ) + parser.add_argument( + "--mock", action="store_true", help="Mock data mode" + ) + + return parser.parse_args() - for i, argv in enumerate(sys.argv[6:], start=6): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {argv}") - else: - print(f" argv[{i}] (unexpected): {argv}") + +if __name__ == "__main__": + args = parse_args() + + try: + print("=" * 60) + print("COSMOSIS_FITTING.PY") + print("=" * 60) + print(f"cosmosis_root: {args.cosmosis_root}") + print(f"data_dir: {args.data_dir}") + print(f"nz_file: {args.nz_file}") + print(f"out_file: {args.out_file}") + print(f"xi files: {args.xi}") + print(f"cov_xi: {args.cov_xi}") + print(f"use_rho_tau: {args.use_rho_tau}") + if args.use_rho_tau: + print(f"rho_stats: {args.rho_stats}") + print(f"tau_stats: {args.tau_stats}") + print(f"cov_tau: {args.cov_tau}") + if args.cl_file: + print(f"cl_file: {args.cl_file}") print("=" * 60) print() - cl_root = sys.argv[6] - gaussian_part = sys.argv[7] # "iNKA" or "OneCovariance" - assert gaussian_part in ["iNKA", "OneCovariance"], "gaussian_part must be either 'iNKA' or 'OneCovariance'" + if args.use_rho_tau and not all([args.rho_stats, args.tau_stats, args.cov_tau]): + raise ValueError( + "--use-rho-tau requires: --rho-stats, --tau-stats, --cov-tau" + ) - pseudo_cl_file = output_folder+'/pseudo_cl_'+cl_root+'.fits' - pseudo_cl_cov_file = output_folder+f'/pseudo_cl_cov_g_ng_{gaussian_part}_{cl_root}.fits' + os.makedirs(args.data_dir, exist_ok=True) + output_dir = os.path.dirname(args.out_file) + os.makedirs(output_dir, exist_ok=True) - print("Creating 2PT fits extension...\n") - if not os.path.exists(pseudo_cl_file): - raise FileNotFoundError("Pseudo-Cl file not found. Please run cosmo_val.py first.") - pseudo_cl_hdu = pseudo_cl_to_fits(pseudo_cl_file) - print("Creating CovMat fits extension...\n") - cov_hdu = cov_pseudo_cl_to_fits(pseudo_cl_cov_file) + print("Loading xi correlation functions...") + if args.mock: + xip_hdu, xim_hdu = treecorr_to_fits(args.xi[0], args.xi[1]) + else: + xip_hdu, xim_hdu = parse_combined_xi_fits(args.xi[0]) + + xi_theta = xip_hdu.data["ANG"] + print(f"Loaded xi: {len(xip_hdu.data)} bins") + + print("Loading covariance matrix...") + cov_hdu = covdat_to_fits(args.cov_xi, filename_cov_tau=None) + print(f"Loaded covariance: shape {cov_hdu.data.shape}") + + print("Loading n(z)...") + nz_hdu = nz_to_fits(args.nz_file) + print("Loaded n(z)") + + cl_ee_hdu = None + cl_bb_hdu = None + if args.cl_file: + print("Loading Cl data...") + if args.cl_file.endswith(".npy"): + ell, cl_ee, cl_bb = load_glass_cl(args.cl_file) + cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) + print(f"Loaded Cl: {len(ell)} multipoles") + else: + raise NotImplementedError(f"Cl format not supported: {args.cl_file}") + + rho_hdu = None + tau_0_p_hdu = None + tau_2_p_hdu = None + if args.use_rho_tau: + print("Loading rho/tau statistics...") + tau_stats = fits.getdata(args.tau_stats) + rho_stats = fits.getdata(args.rho_stats) + tau_theta = tau_stats["theta"] + rho_theta = rho_stats["theta"] + + check_meanr_consistency(xi_theta, tau_theta, rho_theta, threshold=5.0) + print("✓ Forcing rho and tau to use xi meanr values for consistency") - else: - arg_names += ["xi_root", "rhotau_stats", "tau_root", "covmat"] + rho_hdu = rho_to_fits(args.rho_stats, theta=xi_theta) + tau_0_p_hdu, tau_2_p_hdu = tau_to_fits(args.tau_stats, theta=xi_theta) + print("Loaded rho/tau statistics") - for i, argv in enumerate(sys.argv[6:], start=6): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {argv}") - else: - print(f" argv[{i}] (unexpected): {argv}") - print("=" * 60) + cov_hdu = covdat_to_fits(args.cov_xi, filename_cov_tau=args.cov_tau) + + pri_hdr = fits.Header() + pri_hdu = fits.PrimaryHDU(header=pri_hdr) + + print("Assembling FITS file...") + hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu] + + if args.cl_file: + hdu_list.extend([cl_ee_hdu, cl_bb_hdu]) + + if args.use_rho_tau: + hdu_list.extend([tau_0_p_hdu, tau_2_p_hdu, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(args.out_file, overwrite=True) + print(f"✓ FITS file written to {args.out_file}") print() - xi_root = sys.argv[6] - rhotau_stats = sys.argv[7] - tau_root = sys.argv[8] - cov_xi_file = sys.argv[9] #in cosmocov combined txt format - - two_pt_file_xip = output_folder+'/xi_plus_'+xi_root+'.fits' - two_pt_file_xim = output_folder+'/xi_minus_'+xi_root+'.fits' - #in cosmocov format - - use_tau_stats = sys.argv[7] - use_tau_stats = True if use_tau_stats == 'y' else False - rho_stats_file = output_folder+'/rho_tau_stats/rho_stats_'+tau_root+'.fits' if use_tau_stats else None - tau_stats_file = output_folder +'/rho_tau_stats/tau_stats_'+tau_root+'.fits' if use_tau_stats else None - cov_tau_file = output_folder + '/rho_tau_stats/cov_tau_'+tau_root+'_th.npy' if use_tau_stats else None - - - #create the required FITS extensions - print("Creating 2PT fits extension...\n") - if not (os.path.exists(two_pt_file_xip) and os.path.exists(two_pt_file_xim)): - raise FileNotFoundError("2pt files not found. Please run cosmo_val.py first.") - xip_hdu, xim_hdu = treecorr_to_fits(two_pt_file_xip, two_pt_file_xim) - - # Extract xi meanr for consistency enforcement - xi_theta = xip_hdu.data['ANG'] # xi uses 'ANG' column for meanr - - if use_tau_stats and not os.path.exists(tau_stats_file): - raise Warning("Tau stats file not found. Please run cosmo_val.py first. Creating the FITS file without rho and tau statistics.") - use_tau_stats= False - cov_tau_file = None - if use_tau_stats: - - # Load original theta values for validation - tau_stats = fits.getdata(str(tau_stats_file)) - rho_stats = fits.getdata(str(rho_stats_file)) - tau_theta = tau_stats['theta'] - rho_theta = rho_stats['theta'] - - # Validate theta consistency and report differences - print("=" * 60) - print("MEANR CONSISTENCY CHECK") - print("=" * 60) - - # Calculate relative differences - tau_diff = np.abs((tau_theta - xi_theta) / xi_theta) * 100 - rho_diff = np.abs((rho_theta - xi_theta) / xi_theta) * 100 - - print(f"Xi theta range: {xi_theta.min():.6f} - {xi_theta.max():.6f} arcmin") - print(f"Tau theta range: {tau_theta.min():.6f} - {tau_theta.max():.6f} arcmin") - print(f"Rho theta range: {rho_theta.min():.6f} - {rho_theta.max():.6f} arcmin") - print() - print(f"Max tau-xi relative difference: {tau_diff.max():.3f}%") - print(f"Mean tau-xi relative difference: {tau_diff.mean():.3f}%") - print(f"Max rho-xi relative difference: {rho_diff.max():.3f}%") - print(f"Mean rho-xi relative difference: {rho_diff.mean():.3f}%") - - # Check for excessive differences - max_allowed_diff = 5.0 # 5% threshold - if tau_diff.max() > max_allowed_diff: - raise ValueError(f"Tau-xi meanr difference exceeds {max_allowed_diff}%: {tau_diff.max():.3f}%") - if rho_diff.max() > max_allowed_diff: - raise ValueError(f"Rho-xi meanr difference exceeds {max_allowed_diff}%: {rho_diff.max():.3f}%") - - print(f"✓ All differences below {max_allowed_diff}% threshold") - print("✓ Forcing rho and tau to use xi meanr values for consistency") - print("=" * 60) - print() - - print('Creating rho stats fits extension...\n') - rho_hdu = rho_to_fits(rho_stats_file) - print("Creating tau fits extensions...\n") - tau_0_p_hdu, tau_2_p_hdu = tau_to_fits(tau_stats_file) - print("Creating CovMat fits extension...\n") - cov_hdu = covdat_to_fits(cov_xi_file, cov_tau_file) - print("Creating n(z) fits extension...\n") - nz_hdu = nz_to_fits(nz_file) - - # create header for primary HDU - pri_hdr_dict = {} - - pri_hdr = fits.Header() - for key in pri_hdr_dict: - pri_hdr[key] = pri_hdr_dict[key] - - # create primary HDU - pri_hdu = fits.PrimaryHDU(header=pri_hdr) - - # create final FITS HDU - print("Writing out combined FITS file...\n") - if use_pseudo_cell: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, pseudo_cl_hdu] - else: - if use_tau_stats: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu, tau_0_p_hdu, tau_2_p_hdu, rho_hdu] - else: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu] - hdul = fits.HDUList(hdu_list) - hdul.writeto(out_file, overwrite=True) - print("FITS file written out to %s" % out_file) + generate_cosmosis_config(args) + + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) From ecf3769f20d6b1de2e72ced847b6cbf99c1d14e2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 27 Oct 2025 23:39:51 +0100 Subject: [PATCH 19/34] remove defunct pipeline.sh --- cosmo_inference/pipeline.sh | 127 ------------------------------------ 1 file changed, 127 deletions(-) delete mode 100755 cosmo_inference/pipeline.sh diff --git a/cosmo_inference/pipeline.sh b/cosmo_inference/pipeline.sh deleted file mode 100755 index 8d20cb9..0000000 --- a/cosmo_inference/pipeline.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -# Transform long options to short ones -for arg in "$@"; do - shift - case "$arg" in - '--help') set -- "$@" '-h' ;; - '--pcf') set -- "$@" '-p' ;; - '--covmat') set -- "$@" '-c' ;; - '--inference') set -- "$@" '-i' ;; - '--mcmc_process') set -- "$@" '-m' ;; - *) set -- "$@" "$arg" ;; - esac -done - -# Parse short options -OPTIND=1 -while getopts "hpcim" opt -do - case "$opt" in - 'h') - echo "Please input a flag: --help, --pcf, --covmat, --inference or --mcmc_process "; - exit 0 - ;; - 'p') - echo "Running cosmo_val.py to calculate 2 point correlation functions"; - python notebooks/cosmo_val/cosmo_val.py - ;; - 'c') - read -p 'COVARIANCE FILE: ' covmat_file; - read -p 'OUTPUT STUB (without extension): ' output_stub; - echo "Processing covariance matrix"; - python scripts/cosmocov_process.py $covmat_file $output_stub - ;; - 'i') - read -p 'COSMOSIS ROOT: ' cosmosis_root; - - # Create the folder if it does not exist - if [ ! -d "data/$cosmosis_root" ]; then - mkdir -p "data/$cosmosis_root" - echo "Directory 'data/$cosmosis_root' created." - else - echo "Directory 'data/$cosmosis_root' already exists." - fi - - read -p 'COSMO_VAL OUTPUT FOLDER: ' output_folder; - read -p 'NZ FILE:' nz_file; - read -p 'OUTPUT MCMC CHAIN FOLDER: ' data; - read -p 'USE PSEUDO_CELL? (y/n): ' pseudo_cell; - - if [ "${pseudo_cell}" == "y" ]; then - - read -p 'CL ROOT: ' cl_root; - read -p 'GAUSSIAN COVARIANCE? (iNKA/OneCovariance): ' gaussian_part; - - - echo "Using pseudo cell" - out_file="data/${cosmosis_root}/cosmosis_${cosmosis_root}_cell.fits" - - python scripts/cosmosis_fitting.py $cosmosis_root $output_folder $nz_file $pseudo_cell $out_file $cl_root $gaussian_part; - else - read -p 'XI ROOT: ' xi_root; - read -p 'USE RHO/TAU_STATS? (y/n): ' rhotau_stats; - echo $rhotau_stats - read -p 'TAU ROOT: ' tau_root; - read -p 'COV_XI MAT TXT FILE:' covmat; - - out_file="data/${cosmosis_root}/cosmosis_${cosmosis_root}.fits"; - - #LG: add check if xi_plus/xi_minus fits file exists - python scripts/cosmosis_fitting.py $cosmosis_root $output_folder $nz_file $pseudo_cell $out_file $xi_root $rhotau_stats $tau_root $covmat; - - fi - - if [ "${pseudo_cell}" == "y" ]; then - output_ini_file="cosmosis_config/cosmosis_pipeline_${cosmosis_root}_cell.ini" - cp cosmosis_config/cosmosis_pipeline_A_ia_cell.ini $output_ini_file - else - output_ini_file="cosmosis_config/cosmosis_pipeline_${cosmosis_root}.ini" - if [ "${rhotau_stats}" == "y" ]; then - cp cosmosis_config/cosmosis_pipeline_A_psf.ini $output_ini_file; - else - cp cosmosis_config/cosmosis_pipeline_A_ia.ini $output_ini_file; - fi - fi - - sed -i "/^\[DEFAULT\]/a\SCRATCH = ${data}" $output_ini_file; - sed -i "/^\[DEFAULT\]/a\FITS_FILE = ${out_file}" $output_ini_file; - if [ "${pseudo_cell}" == "y" ]; then - sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${cosmosis_root}_cell/samples_${cosmosis_root}_cell.txt" $output_ini_file; - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=CELL_EE" $output_ini_file; - sed -i "/^\[polychord\]/a\polychord_outfile_root = ${cosmosis_root}_cell" $output_ini_file; - sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${cosmosis_root}_cell" $output_ini_file; - else - sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${cosmosis_root}/samples_${cosmosis_root}.txt" $output_ini_file; - if [ "${rhotau_stats}" == "y" ]; then - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_psf.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors_psf.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS" $output_ini_file; - sed -i "/^\[2pt_like]/a\add_xi_sys=T" $output_ini_file; - else - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS" $output_ini_file; - fi - sed -i "/^\[polychord\]/a\polychord_outfile_root = ${root}" $output_ini_file; - sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${root}" $output_ini_file; - fi - echo "Prepared CosmoSIS configuration file in $output_ini_file"; - echo "You can now run the inference with the command: cosmosis $output_ini_file" - ;; - 'm') - # LG: also convert this into a script to directly output contour plots - echo "Run the cosmo_inference/notebooks/MCMC.ipynb notebook to analyse your chains" - ;; - '?') - print_usage >&2; - exit 1 - ;; - esac -done -shift $(expr $OPTIND - 1) # remove options from positional parameters \ No newline at end of file From e76ba2c16a74d2329b0e5aeea827b27932c18bc2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 00:21:33 +0100 Subject: [PATCH 20/34] add example call --- cosmo_inference/scripts/cosmosis_fitting.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 6c2fc8f..f0328d3 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -409,7 +409,23 @@ def parse_args(): """Parse command-line arguments for unified data/mock interface.""" parser = argparse.ArgumentParser( description="Prepare CosmoSIS inference FITS files from real or mock data. " - "Supports multiple xi input formats (separate files, combined FITS)." + "Supports multiple xi input formats (separate files, combined FITS).", + epilog=""" +Example for SP v1.4.6_A: + python cosmo_inference/scripts/cosmosis_fitting.py \\ + --cosmosis-root "SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ + --data-dir "/n09data/guerrini/output_chains" \\ + --nz-file "/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt" \\ + --out-file "cosmo_inference/data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --xi "notebooks/cosmo_val/output/xi_plus_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + "notebooks/cosmo_val/output/xi_minus_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-xi "cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_processed.txt" \\ + --use-rho-tau \\ + --rho-stats "notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --tau-stats "notebooks/cosmo_val/output/rho_tau_stats/tau_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-tau "notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" + """, + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument( From ddfb617dfa75cc9d5ce8946da74536e317632b35 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:35:59 +0100 Subject: [PATCH 21/34] bugfix and readme --- cosmo_inference/README.md | 38 +++++++++++++++------ cosmo_inference/scripts/cosmosis_fitting.py | 10 ++++-- src/sp_validation/rho_tau.py | 18 +++++----- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index 4da2aaa..eec8043 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -7,20 +7,38 @@ This folder contains the files neccessary to run the cosmological inference pipe To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/) and [CosmoCov](https://github.com/CosmoLike/CosmoCov). To PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. ### To Run -Run the bash script within this folder +The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: +```bash +snakemake --profile slurm -j inference_fiducial ``` -$ ./pipeline.sh -``` -with one of the following flags: - -`--pcf`: This step runs the `cosmo_val.py` script to calculate the various 2 point correlation functions. It will also write the $\xi_{pm}$ correlation functions in a fits file. -`--covmat`: The covariance matrix is calculated here using CosmoCov, by reading in the `./cosmocov_config/cosmocov_{output_root}.ini` file. **Hence make sure the `output_root` here corresponds to the one entered in the prompt**. - -`--inference`: This step writes out the relevant `./cosmosis_config/cosmosis_{output_root}.ini` file, in order to run CosmoSIS to conduct the cosmological inference. It also combines the data needed by CosmoSIS: the $\xi_{pm}$ fits files calculated in `cosmo_val.py`, the covariance matrix, and the nz catalogue, into a single `.fits` file. It also fetches the rho-statistics computed in `cosmo_val.py` to marginalize on PSF leakage parameters. To do so, specify the path to the `cosmo_val` outputs. Note that the Rho- and Tau-statistics have to be stored in a folder `rho_tau_stats` in this folder. If the 2PCF $\xi_{pm}$ are not found it will raise an `Error` to run `cosmo_val`. If the Rho- and Tau-statistics do not exist, the script will raise a `Warning` to run `cosmo_val` but will create a data vector without the Rho- and Tau-statistics. +This will automatically execute all steps: +1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` +2. Compute covariance matrices using CosmoCov +3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` +4. Run CosmoSIS inference + +For standalone FITS data preparation, you can also use the Python script directly: + +```bash +python scripts/cosmosis_fitting.py \ + --cosmosis-root "catalog_version_config" \ + --data-dir "/path/to/output/chains" \ + --nz-file "/path/to/nz_file.txt" \ + --out-file "/path/to/output.fits" \ + --xi "/path/to/xi_plus.fits" "/path/to/xi_minus.fits" \ + --cov-xi "/path/to/covariance.txt" \ + --use-rho-tau \ + --rho-stats "/path/to/rho_stats.fits" \ + --tau-stats "/path/to/tau_stats.fits" \ + --cov-tau "/path/to/cov_tau.npy" +``` -`--mcmc_process`: You can finally analyse the chains with the `MCMC.ipynb` notebook. +You can view all available options with: +```bash +python scripts/cosmosis_fitting.py --help +``` This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue. \ No newline at end of file diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index f0328d3..97659dd 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -234,7 +234,7 @@ def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): "NAME_0": "XI_PLUS", "STRT_0": 0, "NAME_1": "XI_MINUS", - "STRT_1": int(len(covmat) / 2), + "STRT_1": int(len(covmat_xi) / 2), } filename_cov_tau and cov_dict.update({ @@ -518,9 +518,13 @@ def parse_args(): print("Loading xi correlation functions...") if args.mock: - xip_hdu, xim_hdu = treecorr_to_fits(args.xi[0], args.xi[1]) - else: + # Mock mode expects a single combined FITS file xip_hdu, xim_hdu = parse_combined_xi_fits(args.xi[0]) + else: + # Data mode expects two TreeCorr files (xi_plus, xi_minus) + if len(args.xi) != 2: + raise ValueError(f"Data mode requires exactly 2 xi files, got {len(args.xi)}") + xip_hdu, xim_hdu = treecorr_to_fits(args.xi[0], args.xi[1]) xi_theta = xip_hdu.data["ANG"] print(f"Loaded xi: {len(xip_hdu.data)} bins") diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 2647b37..2ba9ee4 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -597,12 +597,12 @@ def get_samples_emcee( psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") - # Check if the path exists - sample_file_path = psf_fitter.get_sample_file_path(version) + # Check if the path exists (use base for cache key to account for different TreeCorr configs) + sample_file_path = psf_fitter.get_sample_file_path(base) if os.path.exists(sample_file_path): print(f"Skipping sampling; {sample_file_path} exists.") - flat_samples = psf_fitter.load_samples(version) - mcmc_result, q = psf_fitter.get_mcmc_from_samples(version) + flat_samples = psf_fitter.load_samples(base) + mcmc_result, q = psf_fitter.get_mcmc_from_samples(base) print(mcmc_result) # Or run MCMC else: @@ -618,7 +618,7 @@ def get_samples_emcee( apply_debias=debias_npatch is not None, savefig="mcmc_samples_" + version + ".png", ) - psf_fitter.save_samples(flat_samples, version) + psf_fitter.save_samples(flat_samples, base) return flat_samples, mcmc_result, q @@ -648,11 +648,11 @@ def get_samples_lsq( psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") - # Check if the path exists - sample_file_path = psf_fitter.get_sample_path(version) + # Check if the path exists (use base for cache key to account for different TreeCorr configs) + sample_file_path = psf_fitter.get_sample_path(base) if os.path.exists(sample_file_path): print(f"Skipping sampling; {sample_file_path} exists.") - flat_samples = psf_fitter.load_samples(version) + flat_samples = psf_fitter.load_samples(base) mcmc_result, q = psf_fitter.get_mcmc_from_samples(flat_samples) print(mcmc_result) # Or run MCMC @@ -666,5 +666,5 @@ def get_samples_lsq( flat_samples, mcmc_result, q = psf_fitter.get_least_squares_params_samples( npatch=debias_npatch, apply_debias=(debias_npatch is not None) ) - psf_fitter.save_samples(flat_samples, version) + psf_fitter.save_samples(flat_samples, base) return flat_samples, mcmc_result, q From a24828259b5a5cbdb2213feb8efb4e68da475720 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:36:20 +0100 Subject: [PATCH 22/34] example ini files for glass mock; not using new priors etc. yet --- .../cosmosis_pipeline_glass_mock_00001.ini | 124 ++++++++++++++++++ ...osmosis_pipeline_glass_mock_00001_cell.ini | 113 ++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini new file mode 100644 index 0000000..ffc0a37 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/glass_mock_chains/glass_mock_00001 +FITS_FILE = data/glass_mock_00001/cosmosis_glass_mock_00001.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/glass_mock_00001 + +[polychord] +polychord_outfile_root = glass_mock_00001 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/glass_mock_00001/samples_glass_mock_00001.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +add_xi_sys=T +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini new file mode 100644 index 0000000..ccb6ecc --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini @@ -0,0 +1,113 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/glass_mock_chains/glass_mock_00001 +FITS_FILE = data/glass_mock_00001/cosmosis_glass_mock_00001.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/glass_mock_00001_cell + + +[polychord] +polychord_outfile_root = glass_mock_00001_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/glass_mock_00001/samples_glass_mock_00001_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE CELL_BB +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like \ No newline at end of file From 55b071968842ed7923cc533a2a01380e550848e3 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:43:06 +0100 Subject: [PATCH 23/34] smarter gitignore --- .gitignore | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2709b40..9b266a6 100644 --- a/.gitignore +++ b/.gitignore @@ -173,5 +173,12 @@ notebooks/demo_add_bands.ipynb CLAUDE.md AGENTS.md -cosmo_inference/cosmosis_config/cosmosis_pipeline_S*.ini +cosmo_inference/cosmosis_config/*.ini cosmo_inference/cosmosis_config/priors_S*.ini +!cosmosis_config/cosmosis_pipeline_A_ia_cell.ini +!cosmosis_config/cosmosis_pipeline_A_ia.ini +!cosmosis_config/cosmosis_pipeline_A_psf.ini +!cosmosis_config/cosmosis_pipeline_A.ini +!cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini +!cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini +!cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini \ No newline at end of file From 0a937c7403773518fd46ea7308465c8b98aec171 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:45:23 +0100 Subject: [PATCH 24/34] fix gitignore, add data ini --- .gitignore | 15 +-- ...sep=1.0_maxsep=250.0_nbins=20_npatch=1.ini | 124 ++++++++++++++++++ 2 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini diff --git a/.gitignore b/.gitignore index 9b266a6..fcd96fc 100644 --- a/.gitignore +++ b/.gitignore @@ -174,11 +174,10 @@ notebooks/demo_add_bands.ipynb CLAUDE.md AGENTS.md cosmo_inference/cosmosis_config/*.ini -cosmo_inference/cosmosis_config/priors_S*.ini -!cosmosis_config/cosmosis_pipeline_A_ia_cell.ini -!cosmosis_config/cosmosis_pipeline_A_ia.ini -!cosmosis_config/cosmosis_pipeline_A_psf.ini -!cosmosis_config/cosmosis_pipeline_A.ini -!cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini -!cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini -!cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini \ No newline at end of file +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini new file mode 100644 index 0000000..d64ac64 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 + +[polychord] +polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +add_xi_sys=T +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file From 508a6520eefa3da9911dabb4a9eba766b706225d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:59:08 +0100 Subject: [PATCH 25/34] make sure cls are included in data fits file --- .gitignore | 3 +- ....0_maxsep=250.0_nbins=20_npatch=1_cell.ini | 113 ++++++++++++++++++ cosmo_inference/scripts/cosmosis_fitting.py | 16 +++ 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini diff --git a/.gitignore b/.gitignore index fcd96fc..156b88e 100644 --- a/.gitignore +++ b/.gitignore @@ -180,4 +180,5 @@ cosmo_inference/cosmosis_config/*.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_A.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini -!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini \ No newline at end of file +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini new file mode 100644 index 0000000..3d28be3 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini @@ -0,0 +1,113 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell + + +[polychord] +polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE CELL_BB +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 97659dd..32bd02d 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -115,6 +115,18 @@ def load_glass_cl(cl_file): return ell, cl_ee, cl_bb +def load_pseudo_cl_fits(cl_file): + """ + Load pseudo-C_ell data stored in FITS format with columns (ELL, EE, EB, BB). + """ + with fits.open(cl_file) as hdul: + data = hdul[1].data + ell = np.asarray(data["ELL"], dtype=np.float64) + cl_ee = np.asarray(data["EE"], dtype=np.float64) + cl_bb = np.asarray(data["BB"], dtype=np.float64) + return ell, cl_ee, cl_bb + + def glass_cl_to_fits(ell, cl_ee, cl_bb): """Convert GLASS C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" nbins = len(ell) @@ -545,6 +557,10 @@ def parse_args(): ell, cl_ee, cl_bb = load_glass_cl(args.cl_file) cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) print(f"Loaded Cl: {len(ell)} multipoles") + elif args.cl_file.endswith(".fits"): + ell, cl_ee, cl_bb = load_pseudo_cl_fits(args.cl_file) + cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) + print(f"Loaded Cl: {len(ell)} multipoles (FITS pseudo-Cl)") else: raise NotImplementedError(f"Cl format not supported: {args.cl_file}") From f9a81792af6b300bc1c4a00bad4d26ac584d4469 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 29 Oct 2025 03:50:28 +0100 Subject: [PATCH 26/34] Update inference configs and pseudo-Cl handling --- .gitignore | 3 +- cosmo_inference/README.md | 10 ++- ....0_maxsep=250.0_nbins=20_npatch=1_cell.ini | 4 +- .../cosmosis_pipeline_glass_mock_00001.ini | 2 +- ...osmosis_pipeline_glass_mock_00001_cell.ini | 6 +- .../cosmosis_config/priors_mock.ini | 11 +++ cosmo_inference/scripts/cosmosis_fitting.py | 80 ++++++++++++++++--- 7 files changed, 96 insertions(+), 20 deletions(-) create mode 100644 cosmo_inference/cosmosis_config/priors_mock.ini diff --git a/.gitignore b/.gitignore index 156b88e..3208c72 100644 --- a/.gitignore +++ b/.gitignore @@ -181,4 +181,5 @@ cosmo_inference/cosmosis_config/*.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini -!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini \ No newline at end of file +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini +!cosmo_inference/cosmosis_config/priors_mock.ini diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index eec8043..f729285 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -19,7 +19,7 @@ This will automatically execute all steps: 3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` 4. Run CosmoSIS inference -For standalone FITS data preparation, you can also use the Python script directly: +For standalone FITS data preparation (real-space inputs plus optional pseudo-$C_\ell$ data), you can also use the Python script directly: ```bash python scripts/cosmosis_fitting.py \ @@ -32,7 +32,9 @@ python scripts/cosmosis_fitting.py \ --use-rho-tau \ --rho-stats "/path/to/rho_stats.fits" \ --tau-stats "/path/to/tau_stats.fits" \ - --cov-tau "/path/to/cov_tau.npy" + --cov-tau "/path/to/cov_tau.npy" \ + --cl-file "/path/to/pseudo_cl.fits" \ + --cov-cl "/path/to/pseudo_cl_cov.fits" ``` You can view all available options with: @@ -40,5 +42,7 @@ You can view all available options with: python scripts/cosmosis_fitting.py --help ``` +Ensure the pseudo-$C_\ell$ spectra (`pseudo_cl_*.fits`) and their covariance (`pseudo_cl_cov_*.fits`) produced by `cosmo_val.py` exist for the requested catalog version (or mock seed) before running the standalone command. -This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue. \ No newline at end of file + +This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue. diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini index 3d28be3..34a7209 100644 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini @@ -105,9 +105,9 @@ corr_type = 0 ; shear_cl -> shear_xi [2pt_like] file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE CELL_BB +data_sets=CELL_EE data_file=%(FITS_FILE)s gaussian_covariance=F -covmat_name=COVMAT +covmat_name=COVMAT_CELL cut_zeros=F like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini index ffc0a37..85374a2 100644 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini @@ -7,7 +7,7 @@ COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library [pipeline] values = cosmosis_config/values_psf.ini -priors = cosmosis_config/priors_psf.ini +priors = cosmosis_config/priors_mock.ini modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like likelihoods = 2pt_like extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini index ccb6ecc..f1a03fd 100644 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini @@ -7,7 +7,7 @@ COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library [pipeline] values = cosmosis_config/values_psf.ini -priors = cosmosis_config/priors_psf.ini +priors = cosmosis_config/priors_mock.ini modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like likelihoods = 2pt_like extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m @@ -105,9 +105,9 @@ corr_type = 0 ; shear_cl -> shear_xi [2pt_like] file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE CELL_BB +data_sets=CELL_EE data_file=%(FITS_FILE)s gaussian_covariance=F -covmat_name=COVMAT +covmat_name=COVMAT_CELL cut_zeros=F like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors_mock.ini b/cosmo_inference/cosmosis_config/priors_mock.ini new file mode 100644 index 0000000..0fe8933 --- /dev/null +++ b/cosmo_inference/cosmosis_config/priors_mock.ini @@ -0,0 +1,11 @@ +[intrinsic_alignment_parameters] +A = gaussian 0.83 0.7 + +[cosmological_parameters] +ombh2 = gaussian 0.0244 0.00038 + +[shear_calibration_parameters] +m1 = gaussian 0.0 0.01 + +[nofz_shifts] +bias_1 = gaussian -0.033 0.013 diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 32bd02d..38a88b0 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -93,8 +93,8 @@ def parse_combined_xi_fits(filepath): xim_vals = xi_data["xim"] xi_theta = xi_data["ANG"] if "ANG" in xi_data.names else xi_data["meanr"] - xip_hdu = _create_2pt_hdu(xip_vals, xi_theta, "XI_PLUS", "P+P", "P+P") - xim_hdu = _create_2pt_hdu(xim_vals, xi_theta, "XI_MINUS", "P+P", "P+P") + xip_hdu = _create_2pt_hdu(xip_vals, xi_theta, "XI_PLUS", "G+R", "G+R") + xim_hdu = _create_2pt_hdu(xim_vals, xi_theta, "XI_MINUS", "G-R", "G-R") return xip_hdu, xim_hdu @@ -127,8 +127,8 @@ def load_pseudo_cl_fits(cl_file): return ell, cl_ee, cl_bb -def glass_cl_to_fits(ell, cl_ee, cl_bb): - """Convert GLASS C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" +def cl_to_fits(ell, cl_ee, cl_bb): + """Convert C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" nbins = len(ell) lst = np.arange(1, nbins + 1) @@ -170,6 +170,40 @@ def glass_cl_to_fits(ell, cl_ee, cl_bb): return cl_ee_hdu, cl_bb_hdu +def cov_cl_to_fits(cov_file, nbins): + """Convert pseudo-C_ell covariance to a CosmoSIS ImageHDU.""" + if cov_file.endswith(".fits"): + with fits.open(cov_file) as hdul: + if "COVAR_EE_EE" in hdul: + cov_data = np.asarray(hdul["COVAR_EE_EE"].data, dtype=np.float64) + else: + cov_data = np.asarray(hdul[0].data, dtype=np.float64) + elif cov_file.endswith(".npy"): + cov_data = np.load(cov_file) + else: + raise NotImplementedError(f"Unsupported pseudo-Cl covariance format: {cov_file}") + + if cov_data.shape[0] != cov_data.shape[1]: + raise ValueError("Pseudo-Cl covariance matrix must be square") + if cov_data.shape[0] != nbins: + raise ValueError( + "Pseudo-Cl covariance dimension does not match C_ell data length" + ) + + cov_hdu = fits.ImageHDU(cov_data, name="COVMAT_CELL") + cov_dict = { + "COVDATA": "True", + "EXTNAME": "COVMAT_CELL", + "NAME_0": "CELL_EE", + "STRT_0": 0, + } + + for key, value in cov_dict.items(): + cov_hdu.header[key] = value + + return cov_hdu + + def tau_to_fits(filename, theta=None): """ Convert tau statistics to FITS format. @@ -352,10 +386,13 @@ def _generate_ini_file( else: like_section = ( "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" - "\ndata_sets=CELL_EE CELL_BB" + "\ndata_sets=CELL_EE" ) + covmat_line = "covmat_name=COVMAT_CELL" if is_harmonic else "covmat_name=COVMAT" + modifications.append((r"^\[2pt_like\]", like_section)) + modifications.append((r"^covmat_name=.*", covmat_line)) poly_section = f"[polychord]\npolychord_outfile_root = {args.cosmosis_root}{suffix}" modifications.append((r"^\[polychord\]", poly_section)) @@ -382,13 +419,18 @@ def generate_cosmosis_config(args): """Generate CosmoSIS INI files (real-space and optional harmonic-space).""" if args.use_rho_tau: template_base_realspace = "cosmosis_pipeline_A_psf.ini" - priors_file = "priors_psf.ini" values_file = "values_psf.ini" else: template_base_realspace = "cosmosis_pipeline_A_ia.ini" - priors_file = "priors.ini" values_file = "values_ia.ini" + if args.mock: + priors_file = "priors_mock.ini" + elif args.use_rho_tau: + priors_file = "priors_psf.ini" + else: + priors_file = "priors.ini" + os.makedirs("cosmosis_config", exist_ok=True) _generate_ini_file( @@ -489,6 +531,12 @@ def parse_args(): required=False, help="Path to C_ell data file (.npy, optional for data and mock)", ) + parser.add_argument( + "--cov-cl", + type=str, + required=False, + help="Path to pseudo-C_ell covariance matrix (required if --cl-file)", + ) parser.add_argument( "--mock", action="store_true", help="Mock data mode" ) @@ -516,6 +564,8 @@ def parse_args(): print(f"cov_tau: {args.cov_tau}") if args.cl_file: print(f"cl_file: {args.cl_file}") + if args.cov_cl: + print(f"cov_cl: {args.cov_cl}") print("=" * 60) print() @@ -523,6 +573,10 @@ def parse_args(): raise ValueError( "--use-rho-tau requires: --rho-stats, --tau-stats, --cov-tau" ) + if args.cl_file and not args.cov_cl: + raise ValueError("--cov-cl is required when --cl-file is provided") + if args.cov_cl and not args.cl_file: + raise ValueError("--cl-file is required when --cov-cl is provided") os.makedirs(args.data_dir, exist_ok=True) output_dir = os.path.dirname(args.out_file) @@ -551,18 +605,21 @@ def parse_args(): cl_ee_hdu = None cl_bb_hdu = None + cov_cl_hdu = None if args.cl_file: print("Loading Cl data...") if args.cl_file.endswith(".npy"): ell, cl_ee, cl_bb = load_glass_cl(args.cl_file) - cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) + cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) print(f"Loaded Cl: {len(ell)} multipoles") elif args.cl_file.endswith(".fits"): ell, cl_ee, cl_bb = load_pseudo_cl_fits(args.cl_file) - cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) + cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) print(f"Loaded Cl: {len(ell)} multipoles (FITS pseudo-Cl)") else: raise NotImplementedError(f"Cl format not supported: {args.cl_file}") + cov_cl_hdu = cov_cl_to_fits(args.cov_cl, len(ell)) + print("Loaded pseudo-Cl covariance") rho_hdu = None tau_0_p_hdu = None @@ -587,7 +644,10 @@ def parse_args(): pri_hdu = fits.PrimaryHDU(header=pri_hdr) print("Assembling FITS file...") - hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu] + hdu_list = [pri_hdu, cov_hdu] + if cov_cl_hdu is not None: + hdu_list.append(cov_cl_hdu) + hdu_list.extend([nz_hdu, xip_hdu, xim_hdu]) if args.cl_file: hdu_list.extend([cl_ee_hdu, cl_bb_hdu]) From e74d3cf49d20800b9bfc69985b7d7e1ab75e5166 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 3 Nov 2025 10:24:32 +0100 Subject: [PATCH 27/34] cosmosis_fitting fixes --- cosmo_inference/scripts/cosmosis_fitting.py | 70 ++++++++------------- 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 38a88b0..350684c 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -99,32 +99,28 @@ def parse_combined_xi_fits(filepath): return xip_hdu, xim_hdu -def load_glass_cl(cl_file): - """ - Load GLASS mock C_ell from .npy file. - Expected shape: (5, 32) where row 0=ell, row 1=EE, row 4=BB - """ - cl_block = np.load(cl_file) - if cl_block.shape[0] < 5: - raise ValueError(f"Unexpected C_ell array shape {cl_block.shape} for {cl_file}") - - ell = np.asarray(cl_block[0], dtype=np.float64) - cl_ee = np.asarray(cl_block[1], dtype=np.float64) - cl_bb = np.asarray(cl_block[4], dtype=np.float64) - - return ell, cl_ee, cl_bb +def load_pseudo_cl(cl_file): + """Load pseudo-C_ell spectra from .npy or FITS formats.""" + if cl_file.endswith(".npy"): + cl_block = np.load(cl_file) + if cl_block.shape[0] < 5: + raise ValueError( + f"Unexpected C_ell array shape {cl_block.shape} for {cl_file}" + ) + ell = np.asarray(cl_block[0], dtype=np.float64) + cl_ee = np.asarray(cl_block[1], dtype=np.float64) + cl_bb = np.asarray(cl_block[4], dtype=np.float64) + return ell, cl_ee, cl_bb + if cl_file.endswith(".fits"): + with fits.open(cl_file) as hdul: + data = hdul[1].data + ell = np.asarray(data["ELL"], dtype=np.float64) + cl_ee = np.asarray(data["EE"], dtype=np.float64) + cl_bb = np.asarray(data["BB"], dtype=np.float64) + return ell, cl_ee, cl_bb -def load_pseudo_cl_fits(cl_file): - """ - Load pseudo-C_ell data stored in FITS format with columns (ELL, EE, EB, BB). - """ - with fits.open(cl_file) as hdul: - data = hdul[1].data - ell = np.asarray(data["ELL"], dtype=np.float64) - cl_ee = np.asarray(data["EE"], dtype=np.float64) - cl_bb = np.asarray(data["BB"], dtype=np.float64) - return ell, cl_ee, cl_bb + raise NotImplementedError(f"Cl format not supported: {cl_file}") def cl_to_fits(ell, cl_ee, cl_bb): @@ -170,14 +166,11 @@ def cl_to_fits(ell, cl_ee, cl_bb): return cl_ee_hdu, cl_bb_hdu -def cov_cl_to_fits(cov_file, nbins): +def cov_cl_to_fits(cov_file, cov_hdu="COVAR_FULL"): """Convert pseudo-C_ell covariance to a CosmoSIS ImageHDU.""" if cov_file.endswith(".fits"): with fits.open(cov_file) as hdul: - if "COVAR_EE_EE" in hdul: - cov_data = np.asarray(hdul["COVAR_EE_EE"].data, dtype=np.float64) - else: - cov_data = np.asarray(hdul[0].data, dtype=np.float64) + cov_data = np.asarray(hdul[cov_hdu].data, dtype=np.float64) elif cov_file.endswith(".npy"): cov_data = np.load(cov_file) else: @@ -185,10 +178,6 @@ def cov_cl_to_fits(cov_file, nbins): if cov_data.shape[0] != cov_data.shape[1]: raise ValueError("Pseudo-Cl covariance matrix must be square") - if cov_data.shape[0] != nbins: - raise ValueError( - "Pseudo-Cl covariance dimension does not match C_ell data length" - ) cov_hdu = fits.ImageHDU(cov_data, name="COVMAT_CELL") cov_dict = { @@ -608,17 +597,10 @@ def parse_args(): cov_cl_hdu = None if args.cl_file: print("Loading Cl data...") - if args.cl_file.endswith(".npy"): - ell, cl_ee, cl_bb = load_glass_cl(args.cl_file) - cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) - print(f"Loaded Cl: {len(ell)} multipoles") - elif args.cl_file.endswith(".fits"): - ell, cl_ee, cl_bb = load_pseudo_cl_fits(args.cl_file) - cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) - print(f"Loaded Cl: {len(ell)} multipoles (FITS pseudo-Cl)") - else: - raise NotImplementedError(f"Cl format not supported: {args.cl_file}") - cov_cl_hdu = cov_cl_to_fits(args.cov_cl, len(ell)) + ell, cl_ee, cl_bb = load_pseudo_cl(args.cl_file) + print(f"Loaded Cl: {len(ell)} multipoles") + cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) + cov_cl_hdu = cov_cl_to_fits(args.cov_cl, cov_hdu="COVAR_FULL") print("Loaded pseudo-Cl covariance") rho_hdu = None From f18d6dda85490de01f47207d7659d6cb60c9ab03 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 3 Nov 2025 16:09:41 +0100 Subject: [PATCH 28/34] better control of output files --- cosmo_inference/scripts/cosmosis_fitting.py | 103 +++++++++++++++----- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 350684c..8ad2167 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -4,6 +4,7 @@ import os import re import sys +from pathlib import Path import numpy as np from astropy.io import fits @@ -335,15 +336,15 @@ def _generate_ini_file( is_harmonic=False, ): """Generate a CosmoSIS INI configuration file from template with modifications.""" - template_path = f"cosmosis_config/{template_base}" - output_path = f"cosmosis_config/cosmosis_pipeline_{args.cosmosis_root}{suffix}.ini" + template_path = Path("cosmosis_config") / template_base + output_path = Path(args.output_config_dir) / f"cosmosis_pipeline_{args.config_name_base}{suffix}.ini" with open(template_path, "r") as f: config_content = f.read() modifications = [] - relative_fits_file = f"data/{args.cosmosis_root}/cosmosis_{args.cosmosis_root}.fits" + relative_fits_file = args.config_relative_fits default_section = ( f"[DEFAULT]\nSCRATCH = {args.data_dir}\nFITS_FILE = {relative_fits_file}" ) @@ -396,6 +397,8 @@ def _generate_ini_file( pattern, replacement, config_content, flags=re.MULTILINE ) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: f.write(config_content) @@ -420,7 +423,7 @@ def generate_cosmosis_config(args): else: priors_file = "priors.ini" - os.makedirs("cosmosis_config", exist_ok=True) + os.makedirs(args.output_config_dir, exist_ok=True) _generate_ini_file( args, @@ -454,21 +457,40 @@ def parse_args(): description="Prepare CosmoSIS inference FITS files from real or mock data. " "Supports multiple xi input formats (separate files, combined FITS).", epilog=""" -Example for SP v1.4.6_A: - python cosmo_inference/scripts/cosmosis_fitting.py \\ - --cosmosis-root "SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ - --data-dir "/n09data/guerrini/output_chains" \\ +Example for SP_v1.4.6_leak_corr (real data): + python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmosis_fitting.py \\ + --cosmosis-root "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ + --data-dir "/n09data/guerrini/output_chains/SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ + --nz-file "/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt" \\ + --output-root "/home/guerrini/sp_validation/cosmo_inference" \\ + --xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/xi_plus_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/xi_minus_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_leak_corr_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked/covariance_SP_v1.4.6_leak_corr_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked_processed.txt" \\ + --use-rho-tau \\ + --rho-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --tau-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/tau_stats_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-tau "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" \\ + --cl-file "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_SP_v1.4.6_leak_corr.fits" \\ + --cov-cl "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_cov_g_ng_iNKA_SP_v1.4.6_leak_corr.fits" + +Example for glass mock v0 (mock data): + python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmosis_fitting.py \\ + --mock \\ + --cosmosis-root "glass_mock_v0_00001" \\ + --data-dir "/n09data/guerrini/glass_mock_chains/glass_mock_v0_00001" \\ --nz-file "/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt" \\ - --out-file "cosmo_inference/data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - --xi "notebooks/cosmo_val/output/xi_plus_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - "notebooks/cosmo_val/output/xi_minus_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - --cov-xi "cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_processed.txt" \\ + --output-root "/home/guerrini/sp_validation/cosmo_inference" \\ + --output-basename "glass_mocks/v0/glass_mock_00001" \\ + --xi "/n09data/guerrini/glass_mock_v1.4.6/results/xi_glass_mock_00001_4096_nbins=20.fits" \\ + --cov-xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked_processed.txt" \\ --use-rho-tau \\ - --rho-stats "notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - --tau-stats "notebooks/cosmo_val/output/rho_tau_stats/tau_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - --cov-tau "notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" + --rho-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --tau-stats "/n17data/cdaley/unions/pure_eb/results/glass_mock_rhotau_samples/00001/tau_stats_sampled.fits" \\ + --cov-tau "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" \\ + --cl-file "/n09data/guerrini/glass_mock_v1.4.6/results/cl_glass_mock_00001_4096.npy" \\ + --cov-cl "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_cov_g_ng_iNKA_SP_v1.4.6.fits" """, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( @@ -478,9 +500,6 @@ def parse_args(): "--data-dir", type=str, required=True, help="Output MCMC chain directory" ) parser.add_argument("--nz-file", type=str, required=True, help="Path to n(z) file") - parser.add_argument( - "--out-file", type=str, required=True, help="Path to output FITS file" - ) parser.add_argument( "--xi", nargs="+", @@ -490,7 +509,6 @@ def parse_args(): parser.add_argument( "--cov-xi", type=str, required=True, help="Xi covariance matrix file" ) - parser.add_argument( "--use-rho-tau", action="store_true", @@ -529,6 +547,24 @@ def parse_args(): parser.add_argument( "--mock", action="store_true", help="Mock data mode" ) + parser.add_argument( + "--output-root", + type=str, + required=True, + help=( + "Cosmo inference root directory (e.g., /home/guerrini/sp_validation/cosmo_inference). " + "FITS and config files are written under this root." + ), + ) + parser.add_argument( + "--output-basename", + type=str, + required=False, + help=( + "Optional override for the output sub-directory/basename inside --output-root. " + "Defaults to --cosmosis-root." + ), + ) return parser.parse_args() @@ -537,13 +573,31 @@ def parse_args(): args = parse_args() try: + output_basename = args.output_basename or args.cosmosis_root + output_root_path = Path(args.output_root).expanduser().resolve() + output_basename_path = Path(output_basename) + data_dir_root = output_root_path / "data" / output_basename_path + config_dir_root = output_root_path / "cosmosis_config" + data_dir_root.mkdir(parents=True, exist_ok=True) + config_dir_root.mkdir(parents=True, exist_ok=True) + out_file_path = data_dir_root / f"cosmosis_{args.cosmosis_root}.fits" + args.output_root = str(output_root_path) + args.output_config_dir = str(config_dir_root) + args.output_basename = str(output_basename_path) + args.config_name_base = str(output_basename_path).replace("/", "_") + args.config_relative_fits = str( + Path("data") / output_basename_path / f"cosmosis_{args.cosmosis_root}.fits" + ) + print("=" * 60) print("COSMOSIS_FITTING.PY") print("=" * 60) print(f"cosmosis_root: {args.cosmosis_root}") print(f"data_dir: {args.data_dir}") print(f"nz_file: {args.nz_file}") - print(f"out_file: {args.out_file}") + print(f"output_root: {args.output_root}") + print(f"output_basename: {args.output_basename}") + print(f"out_file: {out_file_path}") print(f"xi files: {args.xi}") print(f"cov_xi: {args.cov_xi}") print(f"use_rho_tau: {args.use_rho_tau}") @@ -568,9 +622,6 @@ def parse_args(): raise ValueError("--cl-file is required when --cov-cl is provided") os.makedirs(args.data_dir, exist_ok=True) - output_dir = os.path.dirname(args.out_file) - os.makedirs(output_dir, exist_ok=True) - print("Loading xi correlation functions...") if args.mock: # Mock mode expects a single combined FITS file @@ -638,8 +689,8 @@ def parse_args(): hdu_list.extend([tau_0_p_hdu, tau_2_p_hdu, rho_hdu]) hdul = fits.HDUList(hdu_list) - hdul.writeto(args.out_file, overwrite=True) - print(f"✓ FITS file written to {args.out_file}") + hdul.writeto(str(out_file_path), overwrite=True) + print(f"✓ FITS file written to {out_file_path}") print() generate_cosmosis_config(args) From 1567511117f62d8948041339997530dd267dd240 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 16 Dec 2025 18:40:16 +0100 Subject: [PATCH 29/34] fix: use makedirs with exist_ok for output directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Safer directory creation that handles nested paths and race conditions. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .beads/issues.jsonl | 8 ++++++++ src/sp_validation/cosmo_val.py | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .beads/issues.jsonl diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl new file mode 100644 index 0000000..bb8486a --- /dev/null +++ b/.beads/issues.jsonl @@ -0,0 +1,8 @@ +{"id":"sp_validation-15j","title":"Review and commit uncommitted ellipticity-suffix work","description":"7 files with uncommitted changes: cosmo_val.py, rho_tau.py, cosmosis_fitting.py, cosmic shear paper configs. Appear to be bug fixes (bin construction, file naming, covariance).","status":"in_progress","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:10.777988207+01:00","updated_at":"2025-12-16T17:43:30.366589735+01:00"} +{"id":"sp_validation-1en","title":"Merge PR #165 (catalog path standardization)","description":"PR approved by Lisa and Sacha since Oct/Nov. Ready to merge for 6 weeks. Run: gh pr merge 165 --merge","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:16.438050654+01:00","updated_at":"2025-12-16T17:31:16.438050654+01:00","dependencies":[{"issue_id":"sp_validation-1en","depends_on_id":"sp_validation-d4d","type":"blocks","created_at":"2025-12-16T17:31:37.693645344+01:00","created_by":"daemon"}]} +{"id":"sp_validation-d4d","title":"Run test suite and validate","description":"After pytest fix, run full test suite to validate ellipticity-suffix changes.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:13.621333151+01:00","updated_at":"2025-12-16T17:41:14.552278446+01:00","closed_at":"2025-12-16T17:41:14.552278446+01:00","dependencies":[{"issue_id":"sp_validation-d4d","depends_on_id":"sp_validation-es5","type":"blocks","created_at":"2025-12-16T17:31:37.617063707+01:00","created_by":"daemon"}]} +{"id":"sp_validation-es5","title":"Fix pytest config (remove --pydocstyle)","description":"pyproject.toml line 81 has deprecated --pydocstyle flag. Tests won't run until fixed.","status":"closed","priority":2,"issue_type":"bug","created_at":"2025-12-16T17:31:07.933332484+01:00","updated_at":"2025-12-16T17:37:12.452426193+01:00","closed_at":"2025-12-16T17:37:12.452426193+01:00"} +{"id":"sp_validation-gco","title":"Fix pyccl Cosmology.get API deprecation","description":"test_ccl_to_camb_basic and test_camb_backend fail due to pyccl API change. Cosmology object no longer has .get() method.","status":"open","priority":2,"issue_type":"bug","created_at":"2025-12-16T17:41:17.33553874+01:00","updated_at":"2025-12-16T17:41:17.33553874+01:00"} +{"id":"sp_validation-o97","title":"Add nellbins suffix to pseudo-Cl output filenames","description":"For reproducibility, output files should include binning params. Deferred pending binning resolution in PR #168.","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T18:40:03.290961433+01:00","updated_at":"2025-12-16T18:40:03.290961433+01:00"} +{"id":"sp_validation-rq0","title":"Add SP_v1.4.5_glass_mock to test allow-list or fix path","description":"test_catalog_paths_exist fails: SP_v1.4.5_glass_mock shear file missing. Either add to allow-list or update path in cat_config.yaml","status":"open","priority":2,"issue_type":"bug","created_at":"2025-12-16T17:42:59.575066932+01:00","updated_at":"2025-12-16T17:42:59.575066932+01:00"} +{"id":"sp_validation-xop","title":"Coordinate NmtBin linear binning with PR #168","description":"Deferred from ellipticity-suffix. Changes to NmtBin constructor, n_bands vs n_ell_bins, dynamic step. Sacha's PR #168 has 'To be implemented correctly' comment. Coordinate there.","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T18:40:00.538355923+01:00","updated_at":"2025-12-16T18:40:00.538355923+01:00"} diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index d3a46ea..a656efa 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -334,8 +334,7 @@ def ensure_version_exists(ver): if output_dir is not None: cc["paths"]["output"] = output_dir - if not os.path.exists(cc["paths"]["output"]): - os.mkdir(cc["paths"]["output"]) + os.makedirs(cc["paths"]["output"], exist_ok=True) def get_redshift(self, version): """Load redshift distribution for a catalog version. From 14c6c5a7c266c9e189f0bfdd99f0bd74099abb12 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 16 Dec 2025 20:06:28 +0100 Subject: [PATCH 30/34] refactor: cleaner rho/tau path handling and cov skip logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Consolidate Path(outdir) into single outdir_path variable - Add catalog_id with _jk suffix for jackknife covariance files - Fix skip logic: now checks BOTH rho stats and cov exist before skipping (previously would skip rho computation even if cov file was missing) - Unify compute_rho_stats() call instead of separate cov_rho branches 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/sp_validation/rho_tau.py | 47 +++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 2ba9ee4..a7abb93 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -73,6 +73,8 @@ def get_transform(self): def _extract_xip(correlations): """Return flattened array of xip values from a list of correlations.""" return np.array([corr.xip for corr in correlations]).flatten() + + def get_params_rho_tau(cat, survey="other"): # Set parameters @@ -206,26 +208,32 @@ def get_rho_tau( print("Compute Rho and Tau statistics for the version: ", version) start_time = time.time() - rho_path = Path(outdir) / f"rho_stats_{base}.fits" + outdir_path = Path(outdir) + rho_path = outdir_path / f"rho_stats_{base}.fits" + catalog_id = f"{base}_jk" if cov_rho else base + cov_rho_path = ( + outdir_path / f"cov_rho_{catalog_id}.npy" + if cov_rho + else None + ) rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=True ) - if rho_path.exists(): - print(f"Skipping rho statistics computation, file {rho_path} already exists.") - rho_stat_handler.load_rho_stats(rho_path.name) - else: + rho_stats_exists = rho_path.exists() + cov_exists = True if not cov_rho else cov_rho_path.exists() + need_compute = (not rho_stats_exists) or (not cov_exists) + if need_compute: rho_stat_handler.catalogs.set_params(params, outdir) mask = version != "DES" square_size = params["square_size"] - # Build catalogues rho_stat_handler.build_cat_to_compute_rho( config[version]["psf"]["path"], - catalog_id=version, + catalog_id=catalog_id, square_size=square_size, mask=mask, hdu=( @@ -235,20 +243,19 @@ def get_rho_tau( ), ) - if cov_rho: - cov_rho_path = Path(outdir) / f"cov_rho_{base}_jk.npy" - if not cov_rho_path.exists(): - rho_stat_handler.compute_rho_stats( - version, - rho_path.name, - save_cov=True, - func=_extract_xip, - var_method="jackknife", - ) - else: - rho_stat_handler.compute_rho_stats(version, rho_path.name, var_method=None) + rho_stat_handler.compute_rho_stats( + catalog_id, + rho_path.name, + save_cov=cov_rho, + func=_extract_xip if cov_rho else None, + var_method="jackknife" if cov_rho else None, + ) + rho_stat_handler.load_rho_stats(rho_path.name) + else: + print(f"Skipping rho statistics computation, file {rho_path} already exists.") + rho_stat_handler.load_rho_stats(rho_path.name) - tau_path = Path(outdir) / f"tau_stats_{base}.fits" + tau_path = outdir_path / f"tau_stats_{base}.fits" tau_stat_handler = TauStat( catalogs=rho_stat_handler.catalogs, From 92b5da4fff5f7711af4f4f3297f7b4f4d354acf9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 16 Dec 2025 20:07:04 +0100 Subject: [PATCH 31/34] chore: remove deprecated pytest-pydocstyle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plugin no longer compatible with recent pytest versions. Tests now run without pydocstyle validation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f53ecf2..d0ee1d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,6 @@ Homepage = "https://github.com/CosmoStat/sp_validation" test = [ "pytest", "pytest-cov", - "pytest-pydocstyle", "ruff" ] docs = [ @@ -78,7 +77,6 @@ develop = ["sp_validation[test,docs]"] testpaths = ["sp_validation"] addopts = [ "--verbose", - "--pydocstyle", "--cov=sp_validation", "--cov-report=term", "--cov-report=xml", @@ -104,6 +102,3 @@ ignore = [ "E712", # comparison to True should be 'if cond is True:' or 'if cond:' ] -[tool.pydocstyle] -convention = "numpy" -add-ignore = ["D105", "D107"] From b998dc926c62e729be3d76ea3184b6563cbfcf7e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 16 Dec 2025 20:07:26 +0100 Subject: [PATCH 32/34] feat: auto-detect paths and add --template-dir to cosmosis_fitting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add module docstring documenting default paths - Auto-detect cosmo_inference root from script location - Make --output-root optional (defaults to detected root) - Add --template-dir argument for INI template location - Use args.template_dir instead of hardcoded "cosmosis_config" Makes script portable across different checkouts/scratch locations. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- cosmo_inference/scripts/cosmosis_fitting.py | 37 ++++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 8ad2167..6a617d9 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -1,5 +1,13 @@ #!/usr/bin/env python # coding: utf-8 +"""Prepare CosmoSIS inputs from UNIONS validation outputs. + +The script lives in ``cosmo_inference/scripts``. By default it reads templates +from ``cosmo_inference/cosmosis_config`` and writes data products beneath +``cosmo_inference/data`` and ``cosmo_inference/cosmosis_config``. Override +``--template-dir`` or ``--output-root`` to use alternative locations. +""" + import argparse import os import re @@ -336,7 +344,7 @@ def _generate_ini_file( is_harmonic=False, ): """Generate a CosmoSIS INI configuration file from template with modifications.""" - template_path = Path("cosmosis_config") / template_base + template_path = Path(args.template_dir) / template_base output_path = Path(args.output_config_dir) / f"cosmosis_pipeline_{args.config_name_base}{suffix}.ini" with open(template_path, "r") as f: @@ -452,7 +460,15 @@ def generate_cosmosis_config(args): def parse_args(): - """Parse command-line arguments for unified data/mock interface.""" + """Parse command-line arguments for the CosmoSIS preparation workflow. + + The script detects its home directory (``cosmo_inference``) automatically + and uses that as the default anchor for CosmoSIS templates as well as the + generated outputs. Override ``--template-dir`` or ``--output-root`` when + running from a different checkout or writing to scratch space. + """ + cosmo_inference_root = Path(__file__).resolve().parent.parent + parser = argparse.ArgumentParser( description="Prepare CosmoSIS inference FITS files from real or mock data. " "Supports multiple xi input formats (separate files, combined FITS).", @@ -550,10 +566,10 @@ def parse_args(): parser.add_argument( "--output-root", type=str, - required=True, + default=str(cosmo_inference_root), help=( - "Cosmo inference root directory (e.g., /home/guerrini/sp_validation/cosmo_inference). " - "FITS and config files are written under this root." + "Base directory for generated outputs (defaults to the cosmo_inference " + "folder containing this script)." ), ) parser.add_argument( @@ -565,6 +581,15 @@ def parse_args(): "Defaults to --cosmosis-root." ), ) + parser.add_argument( + "--template-dir", + type=str, + default=str(cosmo_inference_root / "cosmosis_config"), + help=( + "Directory containing CosmoSIS template INI files (defaults to the " + "cosmosis_config folder next to this script)." + ), + ) return parser.parse_args() @@ -575,6 +600,7 @@ def parse_args(): try: output_basename = args.output_basename or args.cosmosis_root output_root_path = Path(args.output_root).expanduser().resolve() + template_dir_path = Path(args.template_dir).expanduser().resolve() output_basename_path = Path(output_basename) data_dir_root = output_root_path / "data" / output_basename_path config_dir_root = output_root_path / "cosmosis_config" @@ -584,6 +610,7 @@ def parse_args(): args.output_root = str(output_root_path) args.output_config_dir = str(config_dir_root) args.output_basename = str(output_basename_path) + args.template_dir = str(template_dir_path) args.config_name_base = str(output_basename_path).replace("/", "_") args.config_relative_fits = str( Path("data") / output_basename_path / f"cosmosis_{args.cosmosis_root}.fits" From 7b6cb19defbae28eff7cf34b15014c41b4f3130a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 16 Dec 2025 20:08:01 +0100 Subject: [PATCH 33/34] feat: update cosmic shear paper plots workflow for v1.4.6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Config: - Update from SP_v1.4.5 to SP_v1.4.6 - Point to leak_corr_C_10_80 chain and best-fit outputs - Add direct paths for xi_data, pure_eb_data - Simplify config structure Snakefile: - New plot_xi_bestfit rule for Paper IV figure - Comment out legacy rules (require old config keys) Style: - Minor paper.mplstyle updates 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- cosmo_inference/README.md | 2 +- .../config/config.yaml | 29 ++- .../config/paper.mplstyle | 9 +- .../workflow/Snakefile | 87 ++++----- .../workflow/scripts/plot_xi_bestfit.py | 184 ++++++++++++++++++ 5 files changed, 240 insertions(+), 71 deletions(-) create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/plot_xi_bestfit.py diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index f729285..6da1b94 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -10,7 +10,7 @@ To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosi The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: ```bash -snakemake --profile slurm -j inference_fiducial +snakemake -j inference_fiducial ``` This will automatically execute all steps: diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/config.yaml b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/config.yaml index 219f329..0f27af2 100644 --- a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/config.yaml +++ b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/config.yaml @@ -1,19 +1,18 @@ -version: SP_v1.4.5 -plot_style: config/paper.mplstyle - -ini_dirs: - - /n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/cosmosis_config/pipeline/ - - /home/guerrini/sp_validation/cosmo_inference/cosmosis_config/ +version: SP_v1.4.6 +chain: SP_v1.4.6_leak_corr_C_10_80 +# Data paths +xi_data: /n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/xi_pm_SP_v1.4.6.txt +bestfit_dir: /n09data/guerrini/output_chains/SP_v1.4.6_leak_corr_C_10_80/best_fit/SP_v1.4.6_leak_corr_C_10_80 +pure_eb_data: /n17data/cdaley/unions/pure_eb/results/paper_plots/intermediate/SP_v1.4.6_leak_corr_C_pure_eb_semianalytic.npz -# chain: SP_v1.4.5_C_sc_10_150_new_ia -chain: SP_v1.4.5_A_sc_10_60 +# Scale cuts (arcmin) +scale_min: 10 +scale_max: 80 -npatch: 256 -min_sep: 0.1 -max_sep: 250 -nbins: 20 +# Plot style +plot_style: /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/paper.mplstyle -min_sep_int: 0.08 -max_sep_int: 300 -nbins_int: 100 +# Legacy config for old rules (if needed) +ini_dirs: + - /home/guerrini/sp_validation/cosmo_inference/cosmosis_config/ diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/paper.mplstyle b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/paper.mplstyle index 2c0ba2b..14040dc 100644 --- a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/paper.mplstyle +++ b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/paper.mplstyle @@ -2,12 +2,13 @@ figure.titlesize : 14 figure.titleweight : bold figure.autolayout : True axes.titlesize : 12 -legend.fontsize : 8 +legend.fontsize : 12 lines.linewidth : 1.5 lines.markersize : 4 -axes.labelsize : 10 -xtick.labelsize : 10 -ytick.labelsize : 10 +axes.labelsize : 12 +xtick.labelsize : 12 +ytick.labelsize : 12 text.usetex : True +axes.formatter.use_mathtext : True font.family : serif font.size : 12 \ No newline at end of file diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/Snakefile b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/Snakefile index f87a257..4ca6860 100644 --- a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/Snakefile +++ b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/Snakefile @@ -1,60 +1,45 @@ -configfile: "config/config.yaml" - +import os +from pathlib import Path -def find_cosmosis_inputs(): - chain = config["chain"] - for ini_dir in config["ini_dirs"]: - ini_file = Path(ini_dir) / f"cosmosis_pipeline_{chain}.ini" - if os.path.exists(ini_file): - return ini_file - raise FileNotFoundError( - f"Cosmosis input file not found for chain {chain} in specified directories." - ) +configfile: "config/config.yaml" -rule get_bestfit_model: +rule plot_xi_bestfit: + """Plot ξ± data with best-fit theory from fiducial chain.""" input: - inference_ini=find_cosmosis_inputs(), + xi_data=config["xi_data"], + pure_eb_data=config["pure_eb_data"], output: - bestfit_values_ini=f"results/{config['chain']}/bestfit_values.ini", - cosmosis_ini=f"results/{config['chain']}/cosmosis.ini", - xi_shear=f"results/{config['chain']}/bestfit_xi_shear.txt", - xi_sys=f"results/{config['chain']}/bestfit_xi_sys.txt", + f"/n17data/cdaley/unions/pure_eb/results/paper_plots/{config['version']}_xi_with_bestfit.pdf", params: - bestfit_dir=f"results/{config['chain']}", + bestfit_dir=config["bestfit_dir"], script: - "scripts/get_bestfit_model.py" + "scripts/plot_xi_bestfit.py" -rule paper_plots: - input: - bestfit_xi_shear=f"results/{config['chain']}/bestfit_xi_shear.txt", - bestfit_xi_sys=f"results/{config['chain']}/bestfit_xi_sys.txt", - xi_reporting="/n17data/cdaley/unions/sp_validation/notebooks/cosmo_val/output/{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt".format( - **config - ), - xi_integration="/n17data/cdaley/unions/sp_validation/notebooks/cosmo_val/output/{version}_xi_minsep={min_sep_int}_maxsep={max_sep_int}_nbins={nbins_int}_npatch={npatch}.txt".format( - **config - ), - output: - xis="/n17data/cdaley/unions/pure_eb/results/paper_plots/{version}_eb_with_bestfit_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_xis.pdf".format( - **config - ), - ptes="/n17data/cdaley/unions/pure_eb/results/paper_plots/{version}_eb_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_ptes.pdf".format( - **config - ), - cov="/n17data/cdaley/unions/pure_eb/results/paper_plots/{version}_eb_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_cov.pdf".format( - **config - ), - params: - version=config["version"], - min_sep=lambda params: float(config["min_sep"]), - max_sep=lambda params: int(config["max_sep"]), - nbins=lambda params: int(config["nbins"]), - min_sep_int=lambda params: float(config["min_sep_int"]), - max_sep_int=lambda params: int(config["max_sep_int"]), - nbins_int=lambda params: int(config["nbins_int"]), - npatch=lambda params: int(config["npatch"]), - threads: 1 - script: - "paper_plots.py" +# --- Legacy rules (commented out, require old config keys) --- +# def find_cosmosis_inputs(): +# chain = config["chain"] +# for ini_dir in config["ini_dirs"]: +# ini_file = Path(ini_dir) / f"cosmosis_pipeline_{chain}.ini" +# if os.path.exists(ini_file): +# return ini_file +# raise FileNotFoundError( +# f"Cosmosis input file not found for chain {chain} in specified directories." +# ) +# +# rule get_bestfit_model: +# input: +# inference_ini=find_cosmosis_inputs(), +# output: +# bestfit_values_ini=f"results/{config['chain']}/bestfit_values.ini", +# cosmosis_ini=f"results/{config['chain']}/cosmosis.ini", +# xi_shear=f"results/{config['chain']}/bestfit_xi_shear.txt", +# xi_sys=f"results/{config['chain']}/bestfit_xi_sys.txt", +# params: +# bestfit_dir=f"results/{config['chain']}", +# script: +# "scripts/get_bestfit_model.py" +# +# rule paper_plots: +# ... # References min_sep, max_sep, nbins, etc. from old config diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/plot_xi_bestfit.py b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/plot_xi_bestfit.py new file mode 100644 index 0000000..48194f9 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/plot_xi_bestfit.py @@ -0,0 +1,184 @@ +""" +Plot ξ± data with best-fit theory curve from fiducial cosmology inference. + +Style matches the original paper_plots.py figure: +- Black line: Best-fit ξ_th + ξ_sys +- Blue line: Best-fit ξ_sys alone +- Black points: Data ξ± +- Red points: B-mode ξ_B +- Gray shaded region: excluded by fiducial scale cuts +""" + +import matplotlib.pyplot as plt +import numpy as np +from scipy.interpolate import interp1d + +from snakemake.script import snakemake + +# Load plot style +plt.style.use(snakemake.config["plot_style"]) + +# Fiducial scale cuts from Paper IV (main.tex Sect. 5.5) +# Based on B-modes, PSF leakage, and nonlinear modelling +scale_cut_xip = [9, 83] # arcmin +scale_cut_xim = [16, 83] # arcmin + +# --- Load data ξ± --- +data = np.loadtxt(snakemake.input.xi_data, comments="#") +theta_data = data[:, 1] # meanr in arcmin +xip_data = data[:, 3] +xim_data = data[:, 4] +sigma_xip = data[:, 7] +sigma_xim = data[:, 8] + +# --- Load B-mode data --- +eb_data = np.load(snakemake.input.pure_eb_data) +theta_eb = eb_data["theta"] +xip_B = eb_data["xip_B"] +xim_B = eb_data["xim_B"] +cov_pure_eb = eb_data["cov_pure_eb"] + +# Extract B-mode uncertainties from covariance (blocks 2 and 3) +nbins = len(theta_eb) +sigma_xip_B = np.sqrt(np.diag(cov_pure_eb[2*nbins:3*nbins, 2*nbins:3*nbins])) +sigma_xim_B = np.sqrt(np.diag(cov_pure_eb[3*nbins:4*nbins, 3*nbins:4*nbins])) + +# TreeCorr bin edges: log-spaced from min_sep to max_sep +# These are the actual bin boundaries, not derived from meanr +min_sep, max_sep = 1.0, 250.0 +bin_edges = np.geomspace(min_sep, max_sep, nbins + 1) + +# Nominal bin centers (geometric mean of edges) for matching scale cuts +bin_centers_nominal = np.sqrt(bin_edges[:-1] * bin_edges[1:]) + +# Compute actual bin edge boundaries for scale cuts +def get_bin_edge_cuts(centers, edges, scale_cut): + """Get bin edges that bound the included bins based on nominal centers.""" + mask = (centers >= scale_cut[0]) & (centers <= scale_cut[1]) + idx_first = np.where(mask)[0][0] + idx_last = np.where(mask)[0][-1] + return edges[idx_first], edges[idx_last + 1] + +edge_cut_xip = get_bin_edge_cuts(bin_centers_nominal, bin_edges, scale_cut_xip) +edge_cut_xim = get_bin_edge_cuts(bin_centers_nominal, bin_edges, scale_cut_xim) + +# --- Load best-fit theory --- +bestfit_dir = snakemake.params.bestfit_dir + +# Theory theta in radians, convert to arcmin +theta_theory_rad = np.loadtxt(f"{bestfit_dir}/shear_xi_plus/theta.txt", comments="#") +theta_theory = np.rad2deg(theta_theory_rad) * 60 # radians -> arcmin + +xip_theory = np.loadtxt(f"{bestfit_dir}/shear_xi_plus/bin_1_1.txt", comments="#") +xim_theory = np.loadtxt(f"{bestfit_dir}/shear_xi_minus/bin_1_1.txt", comments="#") + +# Load xi_sys (PSF leakage contribution) +theta_sys_rad = np.loadtxt(f"{bestfit_dir}/xi_sys/theta.txt", comments="#") +theta_sys = np.rad2deg(theta_sys_rad) * 60 +xip_sys = np.loadtxt(f"{bestfit_dir}/xi_sys/shear_xi_plus.txt", comments="#") +xim_sys = np.loadtxt(f"{bestfit_dir}/xi_sys/shear_xi_minus.txt", comments="#") + +# Interpolate to common fine grid +theta_fine = np.geomspace(0.5, 300, 500) +xip_th_interp = interp1d(theta_theory, xip_theory, kind="cubic", fill_value="extrapolate")(theta_fine) +xim_th_interp = interp1d(theta_theory, xim_theory, kind="cubic", fill_value="extrapolate")(theta_fine) +xip_sys_interp = interp1d(theta_sys, xip_sys, kind="cubic", fill_value="extrapolate")(theta_fine) +xim_sys_interp = interp1d(theta_sys, xim_sys, kind="cubic", fill_value="extrapolate")(theta_fine) + +# --- Plotting parameters --- +scale_factor = 1e-4 +xlim = [1, 250] +ylim = [-0.15, 1.25] + +# Smaller markers +ms_data = 3 +ms_bmode = 3 +capsize = 1.5 +elinewidth = 0.8 + +# --- Create figure --- +fig, axes = plt.subplots(1, 2, figsize=(10, 4.5), sharey=True) + +plot_configs = [ + (axes[0], xip_data, sigma_xip, xip_B, sigma_xip_B, xip_th_interp, xip_sys_interp, + edge_cut_xip, r"$\xi_+$", "+"), + (axes[1], xim_data, sigma_xim, xim_B, sigma_xim_B, xim_th_interp, xim_sys_interp, + edge_cut_xim, r"$\xi_-$", "-"), +] + +for idx, (ax, xi_data_arr, sigma_xi, xi_B, sigma_B, xi_th, xi_sys, edge_cut, label, pm) in enumerate(plot_configs): + + show_legend = (idx == 1) # Only right panel + + # Gray out excluded regions (outside fiducial scale cuts, using bin edges) + ax.axvspan(xlim[0], edge_cut[0], color="0.90", zorder=0, alpha=0.7) + ax.axvspan(edge_cut[1], xlim[1], color="0.90", zorder=0, alpha=0.7) + + # Best-fit theory + systematics (black line) + ax.plot( + theta_fine, + theta_fine * (xi_th + xi_sys) / scale_factor, + "-", + color="k", + lw=1.5, + label=r"Best-fit $\xi^{\mathrm{th}}_\pm + \xi^{\mathrm{sys}}_\pm$" if show_legend else None, + zorder=2, + ) + + # Best-fit systematics alone (blue line) + ax.plot( + theta_fine, + theta_fine * xi_sys / scale_factor, + "-", + color="C0", + lw=1.2, + label=r"Best-fit $\xi^{\mathrm{sys}}_\pm$" if show_legend else None, + zorder=2, + ) + + # Data ξ± (black points) - plot at meanr + ax.errorbar( + theta_data, + theta_data * xi_data_arr / scale_factor, + yerr=theta_data * sigma_xi / scale_factor, + fmt="o", + color="k", + markersize=ms_data, + capsize=capsize, + elinewidth=elinewidth, + label=r"$\xi_\pm$" if show_legend else None, + zorder=3, + ) + + # B-mode (red points) - slight x-offset for visibility + theta_eb_offset = theta_eb * 1.03 # 3% offset to right + ax.errorbar( + theta_eb_offset, + theta_eb_offset * xi_B / scale_factor, + yerr=theta_eb_offset * sigma_B / scale_factor, + fmt="o", + color="C3", + markersize=ms_bmode, + capsize=capsize, + elinewidth=elinewidth, + alpha=0.85, + label=r"$\xi^B_\pm$" if show_legend else None, + zorder=3, + ) + + # Zero line + ax.axhline(0, color="gray", linestyle="--", alpha=0.8, linewidth=0.8, zorder=1) + + ax.set_xscale("log") + ax.set_xlim(xlim) + ax.set_ylim(ylim) + ax.set_xlabel(r"$\theta$ (arcmin)") + ax.set_title(label) + if show_legend: + ax.legend(loc="upper left") + +axes[0].set_ylabel(r"$\theta\xi \times 10^4$") + +fig.tight_layout() +fig.savefig(snakemake.output[0], dpi=150, bbox_inches="tight") +print(f"Saved: {snakemake.output[0]}") From 99d24a347ebb71971ebdbd949f2908f0f38762aa Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 16 Dec 2025 20:29:10 +0100 Subject: [PATCH 34/34] chore: remove SP_v1.4.5_glass_mock (data no longer available) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The glass mock data at /n09data/guerrini/glass_mock/results/ no longer exists. v1.4.6_glass_mock remains with valid path_template. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .beads/issues.jsonl | 3 +- notebooks/cosmo_val/cat_config.yaml | 43 ----------------------------- 2 files changed, 2 insertions(+), 44 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index bb8486a..cfb31e0 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,5 +1,6 @@ -{"id":"sp_validation-15j","title":"Review and commit uncommitted ellipticity-suffix work","description":"7 files with uncommitted changes: cosmo_val.py, rho_tau.py, cosmosis_fitting.py, cosmic shear paper configs. Appear to be bug fixes (bin construction, file naming, covariance).","status":"in_progress","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:10.777988207+01:00","updated_at":"2025-12-16T17:43:30.366589735+01:00"} +{"id":"sp_validation-15j","title":"Review and commit uncommitted ellipticity-suffix work","description":"7 files with uncommitted changes: cosmo_val.py, rho_tau.py, cosmosis_fitting.py, cosmic shear paper configs. Appear to be bug fixes (bin construction, file naming, covariance).","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:10.777988207+01:00","updated_at":"2025-12-16T20:08:21.251378337+01:00","closed_at":"2025-12-16T20:08:21.251378337+01:00"} {"id":"sp_validation-1en","title":"Merge PR #165 (catalog path standardization)","description":"PR approved by Lisa and Sacha since Oct/Nov. Ready to merge for 6 weeks. Run: gh pr merge 165 --merge","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:16.438050654+01:00","updated_at":"2025-12-16T17:31:16.438050654+01:00","dependencies":[{"issue_id":"sp_validation-1en","depends_on_id":"sp_validation-d4d","type":"blocks","created_at":"2025-12-16T17:31:37.693645344+01:00","created_by":"daemon"}]} +{"id":"sp_validation-7ww","title":"PR #165: data_base_dir parameter removed (breaking)","description":"CosmologyValidation.__init__() no longer accepts data_base_dir. Paths now absolute in cat_config.yaml. May break external scripts. Approved by Lisa/Sacha - presumably validated for their use cases.","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T20:19:19.480910265+01:00","updated_at":"2025-12-16T20:19:19.480910265+01:00"} {"id":"sp_validation-d4d","title":"Run test suite and validate","description":"After pytest fix, run full test suite to validate ellipticity-suffix changes.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:13.621333151+01:00","updated_at":"2025-12-16T17:41:14.552278446+01:00","closed_at":"2025-12-16T17:41:14.552278446+01:00","dependencies":[{"issue_id":"sp_validation-d4d","depends_on_id":"sp_validation-es5","type":"blocks","created_at":"2025-12-16T17:31:37.617063707+01:00","created_by":"daemon"}]} {"id":"sp_validation-es5","title":"Fix pytest config (remove --pydocstyle)","description":"pyproject.toml line 81 has deprecated --pydocstyle flag. Tests won't run until fixed.","status":"closed","priority":2,"issue_type":"bug","created_at":"2025-12-16T17:31:07.933332484+01:00","updated_at":"2025-12-16T17:37:12.452426193+01:00","closed_at":"2025-12-16T17:37:12.452426193+01:00"} {"id":"sp_validation-gco","title":"Fix pyccl Cosmology.get API deprecation","description":"test_ccl_to_camb_basic and test_camb_backend fail due to pyccl API change. Cosmology object no longer has .get() method.","status":"open","priority":2,"issue_type":"bug","created_at":"2025-12-16T17:41:17.33553874+01:00","updated_at":"2025-12-16T17:41:17.33553874+01:00"} diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index bb791ef..5b1c3ed 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -498,49 +498,6 @@ SP_v1.4.5_faint: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits -SP_v1.4.5_glass_mock: - subdir: /n17data/UNIONS/WL/v1.4.x - pipeline: SP - colour: crimson - getdist_colour: 0.0, 0.5, 1.0 - ls: dashed - marker: d - cov_th: - A: 2782 - n_e: 7.18 - n_psf: 0.752316232272063 - sigma_e: 0.4370966656902571 - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits - path_template: /n09data/guerrini/glass_mock/results/unions_glass_sim_{seed:05d}_4096.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - w_col: w - e1_col: e1 - e1_PSF_col: e1_PSF - e2_col: e2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.6_glass_mock: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP