Standardize catalog paths and add validation guardrails

.beads/issues.jsonl

@@ -0,0 +1,9 @@
+{"id":"sp_validation-15j","title":"Review and commit uncommitted ellipticity-suffix work","description":"7 files with uncommitted changes: cosmo_val.py, rho_tau.py, cosmosis_fitting.py, cosmic shear paper configs. Appear to be bug fixes (bin construction, file naming, covariance).","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:10.777988207+01:00","updated_at":"2025-12-16T20:08:21.251378337+01:00","closed_at":"2025-12-16T20:08:21.251378337+01:00"}
+{"id":"sp_validation-1en","title":"Merge PR #165 (catalog path standardization)","description":"PR approved by Lisa and Sacha since Oct/Nov. Ready to merge for 6 weeks. Run: gh pr merge 165 --merge","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:16.438050654+01:00","updated_at":"2025-12-16T17:31:16.438050654+01:00","dependencies":[{"issue_id":"sp_validation-1en","depends_on_id":"sp_validation-d4d","type":"blocks","created_at":"2025-12-16T17:31:37.693645344+01:00","created_by":"daemon"}]}
+{"id":"sp_validation-7ww","title":"PR #165: data_base_dir parameter removed (breaking)","description":"CosmologyValidation.__init__() no longer accepts data_base_dir. Paths now absolute in cat_config.yaml. May break external scripts. Approved by Lisa/Sacha - presumably validated for their use cases.","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T20:19:19.480910265+01:00","updated_at":"2025-12-16T20:19:19.480910265+01:00"}
+{"id":"sp_validation-d4d","title":"Run test suite and validate","description":"After pytest fix, run full test suite to validate ellipticity-suffix changes.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-16T17:31:13.621333151+01:00","updated_at":"2025-12-16T17:41:14.552278446+01:00","closed_at":"2025-12-16T17:41:14.552278446+01:00","dependencies":[{"issue_id":"sp_validation-d4d","depends_on_id":"sp_validation-es5","type":"blocks","created_at":"2025-12-16T17:31:37.617063707+01:00","created_by":"daemon"}]}
+{"id":"sp_validation-es5","title":"Fix pytest config (remove --pydocstyle)","description":"pyproject.toml line 81 has deprecated --pydocstyle flag. Tests won't run until fixed.","status":"closed","priority":2,"issue_type":"bug","created_at":"2025-12-16T17:31:07.933332484+01:00","updated_at":"2025-12-16T17:37:12.452426193+01:00","closed_at":"2025-12-16T17:37:12.452426193+01:00"}
+{"id":"sp_validation-gco","title":"Fix pyccl Cosmology.get API deprecation","description":"test_ccl_to_camb_basic and test_camb_backend fail due to pyccl API change. Cosmology object no longer has .get() method.","status":"open","priority":2,"issue_type":"bug","created_at":"2025-12-16T17:41:17.33553874+01:00","updated_at":"2025-12-16T17:41:17.33553874+01:00"}
+{"id":"sp_validation-o97","title":"Add nellbins suffix to pseudo-Cl output filenames","description":"For reproducibility, output files should include binning params. Deferred pending binning resolution in PR #168.","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T18:40:03.290961433+01:00","updated_at":"2025-12-16T18:40:03.290961433+01:00"}
+{"id":"sp_validation-rq0","title":"Add SP_v1.4.5_glass_mock to test allow-list or fix path","description":"test_catalog_paths_exist fails: SP_v1.4.5_glass_mock shear file missing. Either add to allow-list or update path in cat_config.yaml","status":"open","priority":2,"issue_type":"bug","created_at":"2025-12-16T17:42:59.575066932+01:00","updated_at":"2025-12-16T17:42:59.575066932+01:00"}
+{"id":"sp_validation-xop","title":"Coordinate NmtBin linear binning with PR #168","description":"Deferred from ellipticity-suffix. Changes to NmtBin constructor, n_bands vs n_ell_bins, dynamic step. Sacha's PR #168 has 'To be implemented correctly' comment. Coordinate there.","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-16T18:40:00.538355923+01:00","updated_at":"2025-12-16T18:40:00.538355923+01:00"}

.gitignore

@@ -171,5 +171,15 @@ notebooks/demo_calibrate_minimal_cat.ipynb
 notebooks/leakage_minimal.ipynb
 notebooks/demo_add_bands.ipynb
 
-cosmo_inference/cosmosis_config/cosmosis_pipeline_S*.ini
-cosmo_inference/cosmosis_config/priors_S*.ini
+CLAUDE.md
+AGENTS.md
+cosmo_inference/cosmosis_config/*.ini
+!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini
+!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini
+!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini
+!cosmo_inference/cosmosis_config/cosmosis_pipeline_A.ini
+!cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini
+!cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini
+!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini
+!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini
+!cosmo_inference/cosmosis_config/priors_mock.ini

cosmo_inference/README.md

@@ -7,20 +7,42 @@ This folder contains the files neccessary to run the cosmological inference pipe
 To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/) and [CosmoCov](https://github.com/CosmoLike/CosmoCov). To PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used.
 
 ### To Run
-Run the bash script within this folder
+The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory:
 
+```bash
+snakemake -j<jobs> inference_fiducial
 ```
-$ ./pipeline.sh
-```
-with one of the following flags:
-
-`--pcf`: This step runs the `cosmo_val.py` script to calculate the various 2 point correlation functions. It will also write the $\xi_{pm}$ correlation functions in a fits file. 
 
-`--covmat`: The covariance matrix is calculated here using CosmoCov, by reading in the `./cosmocov_config/cosmocov_{output_root}.ini` file. **Hence make sure the `output_root` here corresponds to the one entered in the prompt**.
+This will automatically execute all steps:
+1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py`
+2. Compute covariance matrices using CosmoCov
+3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py`
+4. Run CosmoSIS inference
+
+For standalone FITS data preparation (real-space inputs plus optional pseudo-$C_\ell$ data), you can also use the Python script directly:
+
+```bash
+python scripts/cosmosis_fitting.py \
+  --cosmosis-root "catalog_version_config" \
+  --data-dir "/path/to/output/chains" \
+  --nz-file "/path/to/nz_file.txt" \
+  --out-file "/path/to/output.fits" \
+  --xi "/path/to/xi_plus.fits" "/path/to/xi_minus.fits" \
+  --cov-xi "/path/to/covariance.txt" \
+  --use-rho-tau \
+  --rho-stats "/path/to/rho_stats.fits" \
+  --tau-stats "/path/to/tau_stats.fits" \
+  --cov-tau "/path/to/cov_tau.npy" \
+  --cl-file "/path/to/pseudo_cl.fits" \
+  --cov-cl "/path/to/pseudo_cl_cov.fits"
+```
 
-`--inference`: This step writes out the relevant `./cosmosis_config/cosmosis_{output_root}.ini` file, in order to run CosmoSIS to conduct the cosmological inference. It also combines the data needed by CosmoSIS: the $\xi_{pm}$ fits files calculated in `cosmo_val.py`, the covariance matrix, and the nz catalogue, into a single `.fits` file. It also fetches the rho-statistics computed in `cosmo_val.py` to marginalize on PSF leakage parameters. To do so, specify the path to the `cosmo_val` outputs. Note that the Rho- and Tau-statistics have to be stored in a folder `rho_tau_stats` in this folder. If the 2PCF $\xi_{pm}$ are not found it will raise an `Error` to run `cosmo_val`. If the Rho- and Tau-statistics do not exist, the script will raise a   `Warning` to run `cosmo_val` but will create a data vector without the Rho- and Tau-statistics.
+You can view all available options with:
+```bash
+python scripts/cosmosis_fitting.py --help
+``` 
 
-`--mcmc_process`: You can finally analyse the chains with the `MCMC.ipynb` notebook. 
+Ensure the pseudo-$C_\ell$ spectra (`pseudo_cl_*.fits`) and their covariance (`pseudo_cl_cov_*.fits`) produced by `cosmo_val.py` exist for the requested catalog version (or mock seed) before running the standalone command.
 
 
-This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue.
+This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue.

cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini

@@ -0,0 +1,124 @@
+#parameters used elsewhere in this file
+[DEFAULT]
+FITS_FILE = data/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits
+SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1
+COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library
+
+
+[pipeline]
+priors = cosmosis_config/priors_psf.ini
+values = cosmosis_config/values_psf.ini
+modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like 
+likelihoods = 2pt_like
+extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m
+timing = T
+debug = T
+
+[runtime]
+sampler = polychord
+verbosity = debug
+
+[test]
+save_dir = %(SCRATCH)s/test_new_pipeline
+
+[polychord]
+polychord_outfile_root = SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1
+live_points = 192
+feedback = 3
+resume = T
+base_dir = %(SCRATCH)s/polychord
+
+[output]
+filename = %(SCRATCH)s/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt
+format = text
+lock = F
+
+[consistency]
+file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py
+verbose = F
+
+[sample_S8]
+file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py
+
+[camb]
+file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py
+mode=power
+lmax=2508
+feedback=0
+do_reionization=F
+kmin=1e-5
+kmax=20.0
+nk=200
+zmax=5.0
+zmax_background=5.0
+nz_background=500
+halofit_version=mead2020_feedback
+nonlinear=pk
+neutrino_hierarchy=normal
+kmax_extrapolate = 500.0
+
+[load_nz_fits]
+file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py
+nz_file =%(FITS_FILE)s
+data_sets = SOURCE
+
+[photoz_bias]
+file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py
+mode = additive
+sample = nz_source
+bias_section = nofz_shifts
+interpolation = cubic
+output_deltaz_section_name = delta_z_out
+
+[linear_alignment]
+file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py
+method = bk_corrected
+
+[projection]
+file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py
+ell_min_logspaced = 1.0
+ell_max_logspaced = 25000.0
+n_ell_logspaced = 400
+shear-shear = source-source
+shear-intrinsic = source-source
+intrinsic-intrinsic = source-source
+get_kernel_peaks = F
+verbose = F
+
+[add_intrinsic]
+file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py
+shear-shear=T
+position-shear=F
+perbin=F
+
+[2pt_shear]
+file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so
+corr_type = 0  ; shear_cl -> shear_xi
+
+[shear_m_bias]
+file = shear/shear_bias/shear_m_bias.py
+m_per_bin = True
+; Despite the parameter name, this can operate on xi as well as C_ell.
+cl_section = shear_xi_plus shear_xi_minus
+verbose = F
+
+[add_xi_sys]
+file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py
+data_file=%(FITS_FILE)s
+rho_stats_name=RHO_STATS
+
+[tau_from_rho]
+file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py
+data_file=%(FITS_FILE)s
+
+[2pt_like]
+add_xi_sys=T
+data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS
+file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py
+data_file=%(FITS_FILE)s
+gaussian_covariance=F
+covmat_name=COVMAT
+cut_zeros=F
+like_name=2pt_like
+angle_range_XI_PLUS_1_1= 3.0 150.0
+angle_range_XI_MINUS_1_1= 10.0 200.0

cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini

@@ -0,0 +1,124 @@
+#parameters used elsewhere in this file
+[DEFAULT]
+FITS_FILE = data/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits
+SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1
+COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library
+
+
+[pipeline]
+priors = cosmosis_config/priors_psf.ini
+values = cosmosis_config/values_psf.ini
+modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like 
+likelihoods = 2pt_like
+extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m
+timing = T
+debug = T
+
+[runtime]
+sampler = polychord
+verbosity = debug
+
+[test]
+save_dir = %(SCRATCH)s/test_new_pipeline
+
+[polychord]
+polychord_outfile_root = SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1
+live_points = 192
+feedback = 3
+resume = T
+base_dir = %(SCRATCH)s/polychord
+
+[output]
+filename = %(SCRATCH)s/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt
+format = text
+lock = F
+
+[consistency]
+file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py
+verbose = F
+
+[sample_S8]
+file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py
+
+[camb]
+file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py
+mode=power
+lmax=2508
+feedback=0
+do_reionization=F
+kmin=1e-5
+kmax=20.0
+nk=200
+zmax=5.0
+zmax_background=5.0
+nz_background=500
+halofit_version=mead2020_feedback
+nonlinear=pk
+neutrino_hierarchy=normal
+kmax_extrapolate = 500.0
+
+[load_nz_fits]
+file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py
+nz_file =%(FITS_FILE)s
+data_sets = SOURCE
+
+[photoz_bias]
+file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py
+mode = additive
+sample = nz_source
+bias_section = nofz_shifts
+interpolation = cubic
+output_deltaz_section_name = delta_z_out
+
+[linear_alignment]
+file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py
+method = bk_corrected
+
+[projection]
+file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py
+ell_min_logspaced = 1.0
+ell_max_logspaced = 25000.0
+n_ell_logspaced = 400
+shear-shear = source-source
+shear-intrinsic = source-source
+intrinsic-intrinsic = source-source
+get_kernel_peaks = F
+verbose = F
+
+[add_intrinsic]
+file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py
+shear-shear=T
+position-shear=F
+perbin=F
+
+[2pt_shear]
+file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so
+corr_type = 0  ; shear_cl -> shear_xi
+
+[shear_m_bias]
+file = shear/shear_bias/shear_m_bias.py
+m_per_bin = True
+; Despite the parameter name, this can operate on xi as well as C_ell.
+cl_section = shear_xi_plus shear_xi_minus
+verbose = F
+
+[add_xi_sys]
+file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py
+data_file=%(FITS_FILE)s
+rho_stats_name=RHO_STATS
+
+[tau_from_rho]
+file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py
+data_file=%(FITS_FILE)s
+
+[2pt_like]
+add_xi_sys=T
+data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS
+file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py
+data_file=%(FITS_FILE)s
+gaussian_covariance=F
+covmat_name=COVMAT
+cut_zeros=F
+like_name=2pt_like
+angle_range_XI_PLUS_1_1= 3.0 150.0
+angle_range_XI_MINUS_1_1= 10.0 200.0