Efficiency improvements for unit tests

docs/changes/2223.maintenance.md

@@ -0,0 +1 @@
+Efficiency improvements for unit tests (mostly in unit tests of the plotting routines).

src/simtools/layout/telescope_position.py

@@ -1,12 +1,19 @@
 """Telescope positions and coordinate transformations."""
 
+import functools
 import logging
 
 import astropy.units as u
 import numpy as np
 import pyproj
 
 
+@functools.cache
+def _cached_transformer(crs_from, crs_to):
+    """Return a cached pyproj Transformer for the given CRS pair."""
+    return pyproj.Transformer.from_crs(crs_from, crs_to)
+
+
 class InvalidCoordSystemErrorError(Exception):
     """Exception for invalid coordinate system."""
 
@@ -316,7 +323,7 @@ def _convert(self, crs_from, crs_to, xx, yy):
 
         """
         try:
-            transformer = pyproj.Transformer.from_crs(crs_from, crs_to)
+            transformer = _cached_transformer(crs_from, crs_to)
         except pyproj.exceptions.CRSError:
             self._logger.error("Invalid coordinate system")
             raise

tests/unit_tests/conftest.py

@@ -6,7 +6,11 @@
 import mmap
 import os
 import re
+import shutil
+import struct
 import tarfile
+import urllib.error
+import zlib
 from contextlib import ExitStack, contextmanager
 from itertools import chain
 from pathlib import Path
@@ -33,6 +37,39 @@
 logger = logging.getLogger()
 DATABASE_HANDLER_CLASS = db_handler.DatabaseHandler
 
+_REPO_ROOT = Path(__file__).parent.parent.parent
+_GITHUB_RAW_BASE = "https://raw.githubusercontent.com/gammasim/simtools/main/"
+
+
+# Minimal 1x1 white-pixel PNG (67 bytes). Written instead of rendering
+# when savefig() is called with a file path during unit tests.
+def _build_stub_png():
+    """Return bytes of a minimal valid 1x1 white-pixel PNG."""
+
+    def chunk(name, data):
+        crc = struct.pack(">I", zlib.crc32(name + data) & 0xFFFFFFFF)
+        return struct.pack(">I", len(data)) + name + data + crc
+
+    return (
+        b"\x89PNG\r\n\x1a\n"  # PNG signature
+        + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))  # 1x1 RGB
+        + chunk(b"IDAT", zlib.compress(b"\x00\xff\xff\xff"))  # one white pixel
+        + chunk(b"IEND", b"")
+    )
+
+
+_STUB_PNG_BYTES = _build_stub_png()
+
+
+def _local_urlretrieve(url, dest):
+    """Serve local repo files instead of making real HTTP requests in unit tests."""
+    if url.startswith(_GITHUB_RAW_BASE):
+        local_path = _REPO_ROOT / url[len(_GITHUB_RAW_BASE) :]
+        if local_path.exists():
+            shutil.copy(local_path, dest)
+            return dest, None
+    raise urllib.error.HTTPError(url, 404, "Not Found", {}, None)
+
 
 def _is_db_unit_test(request):
     """Return True when the current test carries the db_unit_test marker."""
@@ -121,6 +158,77 @@ def _set_matplotlib_backend():
     plt.switch_backend("Agg")
 
 
+@pytest.fixture(scope="session", autouse=True)
+def _fast_figure_saves():
+    """Skip matplotlib rendering in unit tests for speed.
+
+    - savefig with file paths: writes a minimal PNG stub (skips rendering and encoding)
+    - tight_layout: no-op (layout computation triggers full rendering for text-extent measurement)
+    - colorbar: returns a MagicMock (colorbar creation triggers quad-mesh rendering)
+
+    Non-file outputs (e.g. BytesIO) still render at reduced DPI so live previews work.
+    Unit tests only check file existence and non-zero size; no test reads pixel content.
+    """
+    from unittest.mock import MagicMock
+
+    import matplotlib.figure
+
+    orig_savefig = matplotlib.figure.Figure.savefig
+
+    def _stub_savefig(self, fname, *args, **kwargs):
+        if isinstance(fname, (str, Path)):
+            path = Path(fname)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(_STUB_PNG_BYTES)
+            return None
+        kwargs["dpi"] = min(kwargs.get("dpi", 100), 30)
+        kwargs.pop("bbox_inches", None)
+        return orig_savefig(self, fname, *args, **kwargs)
+
+    def _noop_tight_layout(self, *args, **kwargs):
+        pass
+
+    def _mock_colorbar(self, *args, **kwargs):
+        return MagicMock()
+
+    with mock.patch.object(matplotlib.figure.Figure, "savefig", _stub_savefig):
+        with mock.patch.object(matplotlib.figure.Figure, "tight_layout", _noop_tight_layout):
+            with mock.patch.object(matplotlib.figure.Figure, "colorbar", _mock_colorbar):
+                yield
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _mock_urlretrieve_for_unit_tests():
+    """Prevent real HTTP calls in unit tests by serving local repo files."""
+    with mock.patch("urllib.request.urlretrieve", side_effect=_local_urlretrieve):
+        yield
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _warm_erfa_tables():
+    """Pre-initialise ERFA/IERS tables so the first coordinate-transform test is not penalised.
+
+    The first astropy AltAz→ICRS transform in a fresh process must load ERFA tables from
+    disk (~0.2 s).  Running a dummy transform here during session set-up amortises that
+    cost before any test measures it.  IERS auto-download is disabled so no network call
+    is made.
+    """
+    from astropy.coordinates import AltAz, EarthLocation, SkyCoord
+    from astropy.time import Time
+    from astropy.utils import iers
+
+    prev = iers.conf.auto_download
+    iers.conf.auto_download = False
+    try:
+        loc = EarthLocation(lat=28.76 * u.deg, lon=-17.89 * u.deg, height=2200 * u.m)
+        t = Time("2017-09-16 00:00:00", scale="utc")
+        altaz = AltAz(az=180 * u.deg, alt=45 * u.deg, location=loc, obstime=t)
+        SkyCoord(altaz).icrs
+    finally:
+        iers.conf.auto_download = prev
+    return
+
+
 @pytest.fixture
 def tmp_test_directory(tmpdir_factory):
     """Sets temporary test directories. Some tests depend on this structure."""

tests/unit_tests/data_model/test_metadata_collector.py

@@ -5,7 +5,6 @@
 import json
 import logging
 import re
-import time
 import uuid
 from pathlib import Path
 
@@ -73,7 +72,16 @@ def test_get_data_model_schema_dict(args_dict_site):
     assert metadata.get_data_model_schema_dict() == {}
 
 
-def test_get_top_level_metadata(args_dict_site):
+def test_get_top_level_metadata(args_dict_site, mocker):
+    mocker.patch(
+        "simtools.data_model.metadata_collector.gen.now_date_time_in_isoformat",
+        side_effect=[
+            "2020-01-01T00:00:00+00:00",
+            "2020-01-01T00:00:00+00:00",
+            "2020-01-01T00:00:00+00:00",
+            "2099-01-01T00:00:00+00:00",
+        ],
+    )
     collector = metadata_collector.MetadataCollector(args_dict=args_dict_site)
     assert (
         collector.top_level_meta["cta"]["activity"]["end"]
@@ -85,7 +93,6 @@ def test_get_top_level_metadata(args_dict_site):
     top_level_meta = collector.get_top_level_metadata()
     assert top_level_meta["cta"]["activity"]["end"] == top_level_meta["cta"]["activity"]["start"]
 
-    time.sleep(1)
     collector.observatory = "cta"  # back to default
     top_level_meta = collector.get_top_level_metadata()
     assert top_level_meta["cta"]["activity"]["end"] > top_level_meta["cta"]["activity"]["start"]

tests/unit_tests/data_model/test_schema.py

@@ -131,8 +131,7 @@ def test_validate_dict_using_schema(tmp_test_directory, caplog):
         schema.validate_dict_using_schema(invalid_data, schema_file)
 
 
-@pytest.mark.xfail(reason="No network connection")
-def test_validate_dict_using_schema_remote(tmp_test_directory):
+def test_validate_dict_using_schema_remote(tmp_test_directory, mocker):
     sample_schema = {
         "type": "object",
         "properties": {"name": {"type": "string"}, "age": {"type": "number"}},
@@ -147,13 +146,19 @@ def test_validate_dict_using_schema_remote(tmp_test_directory):
     # sample data dictionary to be validated
     data = {"name": "John", "age": 30}
 
+    mock_url_exists = mocker.patch("simtools.data_model.schema.gen.url_exists")
+
     # with valid meta_schema_url
+    mock_url_exists.return_value = True
     data["meta_schema_url"] = "https://github.com/gammasim/simtools"
     schema.validate_dict_using_schema(data, schema_file)
+    mock_url_exists.assert_called_with("https://github.com/gammasim/simtools")
 
+    mock_url_exists.return_value = False
     data["meta_schema_url"] = "https://invalid_url"
     with pytest.raises(FileNotFoundError, match=r"^Meta schema URL does not exist:"):
         schema.validate_dict_using_schema(data, schema_file)
+    mock_url_exists.assert_called_with("https://invalid_url")
 
 
 def test_validate_schema_astropy_units(caplog):

tests/unit_tests/io/test_ascii_handler.py

@@ -123,7 +123,7 @@ def test_collect_data_dict_from_json():
     assert data["unit"] == "m"
 
 
-def test_collect_data_from_http():
+def test_collect_data_from_http() -> None:
     _file = "src/simtools/schemas/model_parameters/num_gains.schema.yml"
     url = url_simtools
 

tests/unit_tests/production_configuration/test_observation_grid.py

@@ -6,9 +6,23 @@
 from astropy.coordinates import EarthLocation, SkyCoord
 from astropy.tests.helper import assert_quantity_allclose
 from astropy.time import Time
+from astropy.utils import iers
 
 from simtools.production_configuration.observation_grid import ProductionGridEngine
 
+pytestmark = pytest.mark.filterwarnings("ignore::astropy.utils.iers.IERSWarning")
+
+
+@pytest.fixture(autouse=True, scope="module")
+def disable_iers_auto_download():
+    """Disable IERS auto-download during tests to avoid network dependency."""
+    previous_auto_download = iers.conf.auto_download
+    iers.conf.auto_download = False
+    try:
+        yield
+    finally:
+        iers.conf.auto_download = previous_auto_download
+
 
 def test_generate_simulation_grid_keeps_horizontal_coordinates_for_radec_axes():
     axes = {

tests/unit_tests/utils/test_general.py

@@ -220,8 +220,21 @@ def test_is_url():
     assert gen.is_url(5.0) is False
 
 
-@pytest.mark.xfail(reason="No network connection")
-def test_url_exists(caplog):
+def test_url_exists(caplog, mocker):
+    import urllib.error
+
+    def mock_urlopen(url, timeout=5):
+        if url == url_simtools_main:
+            mock_ctx = mocker.MagicMock()
+            mock_ctx.__enter__.return_value.status = 200
+            mock_ctx.__exit__.return_value = False
+            return mock_ctx
+        if url is None:
+            raise AttributeError("'NoneType' object has no attribute")
+        raise urllib.error.URLError("not found")
+
+    mocker.patch("simtools.utils.general.urllib.request.urlopen", side_effect=mock_urlopen)
+
     assert gen.url_exists(url_simtools_main)
     with caplog.at_level(logging.ERROR):
         assert not gen.url_exists(url_simtools)  # raw ULR does not exist

tests/unit_tests/visualization/conftest.py

@@ -1,5 +1,6 @@
 """Shared fixtures for visualization unit tests."""
 
+import matplotlib.figure
 import matplotlib.pyplot as plt
 import pytest
 
@@ -9,3 +10,15 @@ def close_all_figures():
     """Automatically close all matplotlib figures after each test."""
     yield
     plt.close("all")
+
+
+@pytest.fixture(autouse=True)
+def _fast_savefig(monkeypatch):
+    """Override savefig DPI to 10 to speed up visualization unit tests."""
+    original = matplotlib.figure.Figure.savefig
+
+    def fast_savefig(self, fname, *args, **kwargs):
+        kwargs["dpi"] = 10
+        return original(self, fname, *args, **kwargs)
+
+    monkeypatch.setattr(matplotlib.figure.Figure, "savefig", fast_savefig)