From 2b55cc7f30f6c8cddaeea56326e494ab6550d729 Mon Sep 17 00:00:00 2001 From: ndilalla Date: Fri, 8 May 2026 10:51:32 -0700 Subject: [PATCH 1/4] Adding a reload_sources keyword to force non-diffuse source map refresh. --- docs/changelog.rst | 4 ++++ fermipy/gtanalysis.py | 27 ++++++++++++++++++++++++--- fermipy/tests/test_gtanalysis.py | 20 ++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c799413b..53d3ac8d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,10 @@ Changelog This page is a changelog for releases of Fermipy. You can also browse releases on `Github `_. +1.4.2 (unreleased) +------------------ +* Fixed ROI restore consistency in `~fermipy.gtanalysis.GTAnalysis.create`/`load_roi` so residual maps are reproducible immediately after `write_roi` without requiring an additional `optimize()` cycle. + 1.4.1 (02/25/2026) ------------------ * Added tests for macos-15-intel and improved fit stability on macos diff --git a/fermipy/gtanalysis.py b/fermipy/gtanalysis.py index b55f5705..d97af0a1 100644 --- a/fermipy/gtanalysis.py +++ b/fermipy/gtanalysis.py @@ -550,7 +550,8 @@ def files(self): return self._files @classmethod - def create(cls, infile, config=None, params=None, mask=None): + def create(cls, infile, config=None, params=None, mask=None, + restore_strict=True): """Create a new instance of GTAnalysis from an analysis output file generated with `~fermipy.GTAnalysis.write_roi`. By default the new instance will inherit the configuration of the saved @@ -574,6 +575,10 @@ def create(cls, infile, config=None, params=None, mask=None): mask : str Path to a fits file with an updated mask + restore_strict : bool + Regenerate non-diffuse source maps and resync ROI cache + state after loading a saved ROI. + """ infile = os.path.abspath(infile) @@ -587,7 +592,8 @@ def create(cls, infile, config=None, params=None, mask=None): gta = cls(config, validate=validate) gta.setup(init_sources=False) - gta.load_roi(infile, params=params, mask=mask) + gta.load_roi(infile, params=params, mask=mask, + restore_strict=restore_strict) return gta def clone(self, config, **kwargs): @@ -3597,7 +3603,8 @@ def print_model(self, loglevel=logging.INFO): self.logger.log(loglevel, o) - def load_roi(self, infile, reload_sources=False, params=None, mask=None): + def load_roi(self, infile, reload_sources=False, params=None, mask=None, + restore_strict=False): """This function reloads the analysis state from a previously saved instance generated with `~fermipy.gtanalysis.GTAnalysis.write_roi`. @@ -3616,6 +3623,10 @@ def load_roi(self, infile, reload_sources=False, params=None, mask=None): mask : str Path to a fits file with an updated mask + restore_strict : bool + Force regeneration of non-diffuse source maps and refresh + ROI/source cache state from the current likelihood. + """ infile = utils.resolve_path(infile, workdir=self.workdir) @@ -3692,6 +3703,16 @@ def load_roi(self, infile, reload_sources=False, params=None, mask=None): names = [s.name for s in self.roi.sources if not s.diffuse] self.reload_sources(names, False) + if restore_strict: + names = [s.name for s in self.roi.sources if not s.diffuse] + if names: + self.reload_sources(names, init_source=False) + + for name in self.like.sourceNames(): + self._init_source(name) + + self._update_roi() + self.logger.info('Finished Loading ROI') def write_roi(self, outfile=None, diff --git a/fermipy/tests/test_gtanalysis.py b/fermipy/tests/test_gtanalysis.py index 11e5f0b6..9def46be 100644 --- a/fermipy/tests/test_gtanalysis.py +++ b/fermipy/tests/test_gtanalysis.py @@ -202,6 +202,26 @@ def test_gtanalysis_residmap(create_diffuse_dir, create_draco_analysis): make_plots=True) +def test_gtanalysis_create_restore_residmap_consistency(create_diffuse_dir, + create_draco_analysis): + gta = create_draco_analysis + gta.load_roi('fit1') + gta.optimize() + gta.write_roi('restore_consistency_test', make_plots=False) + + ref = gta.residmap(model={}, make_plots=False, + prefix='restore_consistency_ref') + ref_mean = np.nanmean(ref['excess'].data) + + infile = os.path.join(gta.workdir, 'restore_consistency_test.npy') + gta_reloaded = gtanalysis.GTAnalysis.create(infile) + out = gta_reloaded.residmap(model={}, make_plots=False, + prefix='restore_consistency_cmp') + out_mean = np.nanmean(out['excess'].data) + + assert_allclose(out_mean, ref_mean, rtol=1E-3, atol=1E-10) + + #@requires_git_version('02-00-00') def test_gtanalysis_find_sources(create_diffuse_dir, create_draco_analysis): From 950a83e2955fbcd58464cf2bdcd4b78173b043c0 Mon Sep 17 00:00:00 2001 From: ndilalla Date: Fri, 8 May 2026 11:02:04 -0700 Subject: [PATCH 2/4] Minor update. --- docs/changelog.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 53d3ac8d..2d30a645 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,9 @@ releases on `Github `_. 1.4.2 (unreleased) ------------------ -* Fixed ROI restore consistency in `~fermipy.gtanalysis.GTAnalysis.create`/`load_roi` so residual maps are reproducible immediately after `write_roi` without requiring an additional `optimize()` cycle. +* Fixed ROI restore consistency in `~fermipy.gtanalysis.GTAnalysis.create`/`load_roi` +* Added support and tests for the new FL16Y source list +* Fixed some inconsistencies and bugs in the skymap library 1.4.1 (02/25/2026) ------------------ @@ -39,15 +41,16 @@ releases on `Github `_. * Added PS map implementing code from Philippe Bruel in `~fermipy.gtanalysis.GTAnalysis.psmap` * Added PS map visualition in `~fermipy.gtanalysis.plotter.make_psmap_plots` * Added `PS map `_ to the documentation +* Implemented Steve calibration to fix the residmap problem 1.2.2 (01/21/2024) ------------------ -* fix the dependence of scipy due to gammapy +* Fixed the dependence of scipy due to gammapy 1.2.1 (12/08/2023) ------------------ * Small bug fixes. -* pinned astropy<6 +* Temporarily pinned `astropy<6` 1.2 (09/21/2022) ---------------- From c0955d05eaff9880579d8b8efebb79b5f43510f1 Mon Sep 17 00:00:00 2001 From: ndilalla Date: Tue, 12 May 2026 18:14:07 -0700 Subject: [PATCH 3/4] Fixed ROI restore consistency when edisp is true --- docs/changelog.rst | 5 ++- fermipy/gtanalysis.py | 52 ++++++++++++++++++++++++++------ fermipy/tests/test_gtanalysis.py | 28 +++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2d30a645..18f5f29b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,10 @@ releases on `Github `_. 1.4.2 (unreleased) ------------------ -* Fixed ROI restore consistency in `~fermipy.gtanalysis.GTAnalysis.create`/`load_roi` +* Fixed ROI restore consistency in `~fermipy.gtanalysis.GTAnalysis.create` and + `~fermipy.gtanalysis.GTAnalysis.load_roi`: when ``edisp: true`` is set and + ``edisp_disable`` lists diffuse sources, reloading a saved ROI now produces + model counts and residual maps identical to those computed before saving. * Added support and tests for the new FL16Y source list * Fixed some inconsistencies and bugs in the skymap library diff --git a/fermipy/gtanalysis.py b/fermipy/gtanalysis.py index d97af0a1..28403411 100644 --- a/fermipy/gtanalysis.py +++ b/fermipy/gtanalysis.py @@ -7,6 +7,7 @@ import logging import tempfile import filecmp +import xml.etree.ElementTree as ET import time import json from pathlib import Path @@ -576,8 +577,9 @@ def create(cls, infile, config=None, params=None, mask=None, Path to a fits file with an updated mask restore_strict : bool - Regenerate non-diffuse source maps and resync ROI cache - state after loading a saved ROI. + Resynchronize ROI cache state (fixed-model weights, + per-source properties, ROI summary) after loading a saved + ROI. Passed directly to `~fermipy.GTAnalysis.load_roi`. """ @@ -3624,8 +3626,9 @@ def load_roi(self, infile, reload_sources=False, params=None, mask=None, Path to a fits file with an updated mask restore_strict : bool - Force regeneration of non-diffuse source maps and refresh - ROI/source cache state from the current likelihood. + Force deterministic source-map synchronization (including + diffuse/fixed components) and refresh ROI/source cache state + from the current likelihood. """ @@ -3704,9 +3707,8 @@ def load_roi(self, infile, reload_sources=False, params=None, mask=None, self.reload_sources(names, False) if restore_strict: - names = [s.name for s in self.roi.sources if not s.diffuse] - if names: - self.reload_sources(names, init_source=False) + for c in self.components: + c.like.logLike.buildFixedModelWts() for name in self.like.sourceNames(): self._init_source(name) @@ -5508,8 +5510,40 @@ def _create_binned_analysis(self, xmlfile=None, **kwargs): set_edisp_kwargs('gtlike', self.config, kw) self.logger.debug(kw) - self._like = BinnedAnalysis(binnedData=self._obs, - **utils.unicode_to_str(kw)) + # When edisp is enabled globally and some sources have edisp disabled, + # loading an XML that contains apply_edisp='false' on those sources causes + # BinnedAnalysis to set edisp_val=0 instead of -1, leading to incorrect + # model counts after load_roi. Strip the attribute before construction so + # the constructor always starts with edisp_val=-1; set_edisp_flag below + # will then correctly mark those sources as no-edisp. + edisp_disable = [s for s in self.config['gtlike']['edisp_disable'] + if self.roi.has_source(s)] + tmp_srcmdl = None + if edisp_disable and kw.get('edisp_bins', 0) != 0: + try: + tree = ET.parse(kw['srcModel']) + root = tree.getroot() + stripped = False + for src in root.findall('source'): + if src.get('name') in edisp_disable: + spectrum = src.find('spectrum') + if spectrum is not None and 'apply_edisp' in spectrum.attrib: + del spectrum.attrib['apply_edisp'] + stripped = True + if stripped: + fd, tmp_srcmdl = tempfile.mkstemp(suffix='.xml') + os.close(fd) + tree.write(tmp_srcmdl, xml_declaration=True, encoding='unicode') + kw['srcModel'] = tmp_srcmdl + except Exception: + pass + + try: + self._like = BinnedAnalysis(binnedData=self._obs, + **utils.unicode_to_str(kw)) + finally: + if tmp_srcmdl and os.path.exists(tmp_srcmdl): + os.unlink(tmp_srcmdl) # print(self.like.logLike.use_single_fixed_map()) # self.like.logLike.set_use_single_fixed_map(False) diff --git a/fermipy/tests/test_gtanalysis.py b/fermipy/tests/test_gtanalysis.py index 9def46be..9866e743 100644 --- a/fermipy/tests/test_gtanalysis.py +++ b/fermipy/tests/test_gtanalysis.py @@ -212,16 +212,44 @@ def test_gtanalysis_create_restore_residmap_consistency(create_diffuse_dir, ref = gta.residmap(model={}, make_plots=False, prefix='restore_consistency_ref') ref_mean = np.nanmean(ref['excess'].data) + ref_model_means = np.array([np.nanmean(c.model_counts_map().data) + for c in gta.components]) infile = os.path.join(gta.workdir, 'restore_consistency_test.npy') gta_reloaded = gtanalysis.GTAnalysis.create(infile) out = gta_reloaded.residmap(model={}, make_plots=False, prefix='restore_consistency_cmp') out_mean = np.nanmean(out['excess'].data) + out_model_means = np.array([np.nanmean(c.model_counts_map().data) + for c in gta_reloaded.components]) assert_allclose(out_mean, ref_mean, rtol=1E-3, atol=1E-10) + assert_allclose(out_model_means, ref_model_means, rtol=1E-3, atol=1E-10) +def test_gtanalysis_load_roi_restore_strict_same_instance_consistency( + create_diffuse_dir, create_draco_analysis): + gta = create_draco_analysis + gta.load_roi('fit1') + gta.optimize() + gta.write_roi('restore_same_instance_test', make_plots=False) + + ref = gta.residmap(model={}, make_plots=False, + prefix='restore_same_instance_ref') + ref_mean = np.nanmean(ref['excess'].data) + ref_model_means = np.array([np.nanmean(c.model_counts_map().data) + for c in gta.components]) + + gta.load_roi('restore_same_instance_test', restore_strict=True) + out = gta.residmap(model={}, make_plots=False, + prefix='restore_same_instance_cmp') + out_mean = np.nanmean(out['excess'].data) + out_model_means = np.array([np.nanmean(c.model_counts_map().data) + for c in gta.components]) + + assert_allclose(out_mean, ref_mean, rtol=1E-3, atol=1E-10) + assert_allclose(out_model_means, ref_model_means, rtol=1E-3, atol=1E-10) + #@requires_git_version('02-00-00') def test_gtanalysis_find_sources(create_diffuse_dir, create_draco_analysis): From 6a09b4f369c8e61ed66d3fe7df7d01ed454aeb37 Mon Sep 17 00:00:00 2001 From: ndilalla Date: Tue, 12 May 2026 18:24:40 -0700 Subject: [PATCH 4/4] Removing restore_strict argument as no longer needed. --- fermipy/gtanalysis.py | 28 +++------------------------- fermipy/tests/test_gtanalysis.py | 10 +++++----- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/fermipy/gtanalysis.py b/fermipy/gtanalysis.py index 28403411..b0265bd3 100644 --- a/fermipy/gtanalysis.py +++ b/fermipy/gtanalysis.py @@ -551,8 +551,7 @@ def files(self): return self._files @classmethod - def create(cls, infile, config=None, params=None, mask=None, - restore_strict=True): + def create(cls, infile, config=None, params=None, mask=None): """Create a new instance of GTAnalysis from an analysis output file generated with `~fermipy.GTAnalysis.write_roi`. By default the new instance will inherit the configuration of the saved @@ -576,11 +575,6 @@ def create(cls, infile, config=None, params=None, mask=None, mask : str Path to a fits file with an updated mask - restore_strict : bool - Resynchronize ROI cache state (fixed-model weights, - per-source properties, ROI summary) after loading a saved - ROI. Passed directly to `~fermipy.GTAnalysis.load_roi`. - """ infile = os.path.abspath(infile) @@ -594,8 +588,7 @@ def create(cls, infile, config=None, params=None, mask=None, gta = cls(config, validate=validate) gta.setup(init_sources=False) - gta.load_roi(infile, params=params, mask=mask, - restore_strict=restore_strict) + gta.load_roi(infile, params=params, mask=mask) return gta def clone(self, config, **kwargs): @@ -3605,8 +3598,7 @@ def print_model(self, loglevel=logging.INFO): self.logger.log(loglevel, o) - def load_roi(self, infile, reload_sources=False, params=None, mask=None, - restore_strict=False): + def load_roi(self, infile, reload_sources=False, params=None, mask=None): """This function reloads the analysis state from a previously saved instance generated with `~fermipy.gtanalysis.GTAnalysis.write_roi`. @@ -3625,11 +3617,6 @@ def load_roi(self, infile, reload_sources=False, params=None, mask=None, mask : str Path to a fits file with an updated mask - restore_strict : bool - Force deterministic source-map synchronization (including - diffuse/fixed components) and refresh ROI/source cache state - from the current likelihood. - """ infile = utils.resolve_path(infile, workdir=self.workdir) @@ -3706,15 +3693,6 @@ def load_roi(self, infile, reload_sources=False, params=None, mask=None, names = [s.name for s in self.roi.sources if not s.diffuse] self.reload_sources(names, False) - if restore_strict: - for c in self.components: - c.like.logLike.buildFixedModelWts() - - for name in self.like.sourceNames(): - self._init_source(name) - - self._update_roi() - self.logger.info('Finished Loading ROI') def write_roi(self, outfile=None, diff --git a/fermipy/tests/test_gtanalysis.py b/fermipy/tests/test_gtanalysis.py index 9866e743..9f957465 100644 --- a/fermipy/tests/test_gtanalysis.py +++ b/fermipy/tests/test_gtanalysis.py @@ -227,22 +227,22 @@ def test_gtanalysis_create_restore_residmap_consistency(create_diffuse_dir, assert_allclose(out_model_means, ref_model_means, rtol=1E-3, atol=1E-10) -def test_gtanalysis_load_roi_restore_strict_same_instance_consistency( +def test_gtanalysis_load_roi_same_instance_consistency( create_diffuse_dir, create_draco_analysis): gta = create_draco_analysis gta.load_roi('fit1') gta.optimize() - gta.write_roi('restore_same_instance_test', make_plots=False) + gta.write_roi('load_roi_same_instance_test', make_plots=False) ref = gta.residmap(model={}, make_plots=False, - prefix='restore_same_instance_ref') + prefix='load_roi_same_instance_ref') ref_mean = np.nanmean(ref['excess'].data) ref_model_means = np.array([np.nanmean(c.model_counts_map().data) for c in gta.components]) - gta.load_roi('restore_same_instance_test', restore_strict=True) + gta.load_roi('load_roi_same_instance_test') out = gta.residmap(model={}, make_plots=False, - prefix='restore_same_instance_cmp') + prefix='load_roi_same_instance_cmp') out_mean = np.nanmean(out['excess'].data) out_model_means = np.array([np.nanmean(c.model_counts_map().data) for c in gta.components])