Improved version and build info handling:

.env_template

@@ -12,7 +12,7 @@ SIMTOOLS_DB_API_USER=YOUR_USERNAME
 SIMTOOLS_DB_SERVER=cta-simpipe-protodb.zeuthen.desy.de
 SIMTOOLS_DB_SIMULATION_MODEL=CTAO-Simulation-Model
 SIMTOOLS_DB_SIMULATION_MODEL_VERSION=v0.12.0
-SIMTOOLS_SIMTEL_PATH=/workdir/simulation_software/sim_telarray
+SIMTOOLS_SIM_TELARRAY_PATH=/workdir/simulation_software/sim_telarray
 SIMTOOLS_USER_EMAIL=default@email.org
 SIMTOOLS_USER_NAME=simtools-user
 SIMTOOLS_USER_ORCID=0000-0000-0000-0000

.github/workflows/CI-integrationtests.yml

@@ -77,7 +77,7 @@ jobs:
 
       - name: Set sim_telarray path
         run: |
-          echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_ENV"
+          echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIM_TELARRAY_PATH" >> "$GITHUB_ENV"
 
       - name: Create environment file (remote DB; schedule only)
         if: ${{ github.event_name == 'schedule' }}
@@ -87,7 +87,7 @@ jobs:
             echo "SIMTOOLS_DB_API_USER=${{ secrets.DB_API_USER }}"
             echo "SIMTOOLS_DB_API_PW=${{ secrets.DB_API_PW }}"
             echo "SIMTOOLS_DB_API_PORT=${{ secrets.DB_API_PORT }}"
-            echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
+            echo "SIMTOOLS_SIM_TELARRAY_PATH=$(grep '^SIMTOOLS_SIM_TELARRAY_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
             echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
             echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')"
             echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')"
@@ -112,7 +112,7 @@ jobs:
             echo "SIMTOOLS_DB_API_PORT=27017"
             echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}"
             echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}"
-            echo "SIMTOOLS_SIMTEL_PATH=$(grep '^SIMTOOLS_SIMTEL_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
+            echo "SIMTOOLS_SIM_TELARRAY_PATH=$(grep '^SIMTOOLS_SIM_TELARRAY_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
             echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
             echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')"
             echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')"
@@ -163,13 +163,6 @@ jobs:
             --db_simulation_model_version ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} \
             --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH"
 
-      - name: Print versions
-        run: |
-          source /workdir/env/bin/activate
-          pip install --no-cache-dir -e .
-          cat .env
-          simtools-print-version
-
       - name: Integration tests
         shell: bash -l {0}
         env:

.github/workflows/build-simtools-legacy.yml

@@ -140,7 +140,7 @@ jobs:
 
       - name: Set sim_telarray path
         run: |
-          echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIMTEL_PATH" >> "$GITHUB_ENV"
+          echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIM_TELARRAY_PATH" >> "$GITHUB_ENV"
 
       - name: Extract SIMTOOLS_DB_SIMULATION_MODEL
         run: |
@@ -159,7 +159,7 @@ jobs:
             echo "SIMTOOLS_DB_API_PORT=27017"
             echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}"
             echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}"
-            echo "SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray/sim_telarray"
+            echo "SIMTOOLS_SIM_TELARRAY_PATH=/workdir/sim_telarray/sim_telarray"
             echo "SIMTOOLS_CORSIKA_PATH=/workdir/sim_telarray/corsika-run"
           } > .env
 
@@ -180,5 +180,4 @@ jobs:
         run: |
           source /workdir/env/bin/activate
           pip install --no-cache-dir pytest pytest-cov pytest-requirements pytest-xdist
-          simtools-print-version
           pytest --model_version=${{ matrix.model_version }} --no-cov --color=yes -n auto tests/integration_tests/test_applications_from_config.py

docs/changes/1957.api.md

@@ -0,0 +1 @@
+Changed path to `sim_telarray` to `SIMTOOLS_SIM_TELARRAY_PATH` (from `SIMTOOLS_SIMTEL_PATH`). Requires changes by the users to their `.env` files.

docs/changes/1957.feature.md

@@ -0,0 +1,9 @@
+Improved version and build info handling:
+
+- simtools application prints at startup now the simtools, database, CORSIKA, and sim_telarray versions
+- simtools applications output full build information for log level `DEBUG`
+- added two new command line options:
+  - `--build_info` to print detailed build information of the application
+  - `--export_build_info EXPORT_BUILD_INFO` export build information to file (json or yaml format)
+- removed applications `simtools-print-version` (above functionality replaces it)
+- Changed path to `sim_telarray` to `SIMTOOLS_SIM_TELARRAY_PATH` (from `SIMTOOLS_SIMTEL_PATH`). This is consistent with the naming of the simulation software packages and the schema values.

docs/source/user-guide/applications.md

@@ -86,7 +86,6 @@ simtools-plot-tabular-data <applications/simtools-plot-tabular-data>
 simtools-plot-tabular-data-for-model-parameter <applications/simtools-plot-tabular-data-for-model-parameter>
 simtools-plot-simtel-events <applications/simtools-plot-simtel-events>
 simtools-plot-simulated-event-distributions <applications/simtools-plot-simulated-event-distributions>
-simtools-print-version <applications/simtools-print-version>
 simtools-production-derive-corsika-limits <applications/simtools-production-derive-corsika-limits>
 simtools-production-derive-statistics <applications/simtools-production-derive-statistics>
 simtools-production-generate-grid <applications/simtools-production-generate-grid>

pyproject.toml

@@ -110,7 +110,6 @@ scripts.simtools-plot-simtel-events = "simtools.applications.plot_simtel_events:
 scripts.simtools-plot-simulated-event-distributions = "simtools.applications.plot_simulated_event_distributions:main"
 scripts.simtools-plot-tabular-data = "simtools.applications.plot_tabular_data:main"
 scripts.simtools-plot-tabular-data-for-model-parameter = "simtools.applications.plot_tabular_data_for_model_parameter:main"
-scripts.simtools-print-version = "simtools.applications.print_version:main"
 scripts.simtools-production-derive-corsika-limits = "simtools.applications.production_derive_corsika_limits:main"
 scripts.simtools-production-derive-statistics = "simtools.applications.production_derive_statistics:main"
 scripts.simtools-production-generate-grid = "simtools.applications.production_generate_grid:main"

src/simtools/application_control.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 
 import simtools.utils.general as gen
-from simtools import version
+from simtools import dependencies, version
 from simtools.db import db_handler
 from simtools.io import io_handler
 from simtools.settings import config
@@ -89,6 +89,8 @@ def main():
 
     _resolve_model_version_to_latest_patch(args_dict, logger)
 
+    _version_info(args_dict, io_handler_instance, logger)
+
     return ApplicationContext(
         args=args_dict,
         db_config=db_config,
@@ -166,3 +168,29 @@ def resolve(v, k):
 
     resolved = [resolve(v, k) for v, k in zip(versions, kinds)]
     args_dict["model_version"] = resolved if isinstance(mv, list) else resolved[0]
+
+
+def _version_info(args_dict, io_handler_instance, logger):
+    """Print and optionally write version information."""
+    try:
+        build_options = dependencies.get_build_options(args_dict.get("run_time"))
+    except FileNotFoundError:
+        logger.debug("No build options found.")
+        return
+
+    logger.info(
+        f"simtools: {version.__version__} "
+        f"DB: {dependencies.get_database_version_or_name(version=False)} "
+        f"{dependencies.get_database_version_or_name(version=True)} "
+        f"CORSIKA: {build_options.get('corsika_version')} "
+        f"sim_telarray: {build_options.get('simtel_version')}"
+    )
+    logger.debug(f"Build options:\n {build_options}")
+
+    if args_dict.get("export_build_info"):
+        output_path = (
+            io_handler_instance.get_output_file(args_dict["export_build_info"])
+            if io_handler_instance
+            else args_dict["export_build_info"]
+        )
+        dependencies.export_build_info(output_path, args_dict.get("run_time"))

src/simtools/applications/simulate_prod_htcondor_generator.py

@@ -22,7 +22,7 @@
 - Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
   credentials.  These should be listed similarly to a '.env' file and copied to
   'output_path/env.txt'.  Ensure that the path to the simulation software is correctly set to
-  'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
+  'SIMTOOLS_SIM_TELARRAY_PATH=/workdir/sim_telarray'.
 
 To submit jobs, change to the output directory and run:
 

src/simtools/configuration/commandline_parser.py

@@ -106,7 +106,7 @@ def initialize_path_arguments(self):
             required=False,
         )
         _job_group.add_argument(
-            "--simtel_path",
+            "--sim_telarray_path",
             help="path pointing to sim_telarray installation",
             type=Path,
             required=False,
@@ -160,6 +160,18 @@ def initialize_application_execution_arguments(self):
         _job_group.add_argument(
             "--version", action="version", version=f"%(prog)s {simtools.version.__version__}"
         )
+        _job_group.add_argument(
+            "--build_info",
+            action=BuildInfoAction,
+            build_info=f"%(prog)s {simtools.version.__version__}",
+            help="show build information and exit",
+        )
+        _job_group.add_argument(
+            "--export_build_info",
+            help="export build information to file",
+            required=False,
+            type=str,
+        )
 
     def initialize_user_arguments(self):
         """Initialize user arguments."""
@@ -825,3 +837,24 @@ def parse_integer_and_quantity(input_string):
             raise ValueError("Input string does not contain an integer and a astropy quantity.")
 
         return (int(match.group(1)), u.Quantity(float(match.group(2)), match.group(3)))
+
+
+class BuildInfoAction(argparse.Action):
+    """Custom argparse action to display build information."""
+
+    def __init__(self, option_strings, dest=argparse.SUPPRESS, default=argparse.SUPPRESS, **kwargs):
+        """Initialize BuildInfoAction."""
+        self.build_info = kwargs.pop("build_info", "Build information")
+        kwargs.pop("nargs", None)
+        super().__init__(option_strings, dest=dest, default=default, nargs=0, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        """Display build information and exit."""
+        # for efficiency reason, allow import here
+        from simtools import dependencies  # pylint: disable=C0415
+
+        build_options = dependencies.get_build_options()
+        print(f"{self.build_info}")
+        for key, value in build_options.items():
+            print(f"{key}: {value}")
+        parser.exit()

src/simtools/configuration/configurator.py

@@ -2,12 +2,10 @@
 
 import argparse
 import logging
-import os
 import sys
 import uuid
 
 import astropy.units as u
-from dotenv import load_dotenv
 
 import simtools.configuration.commandline_parser as argparser
 from simtools.db.mongo_db import jsonschema_db_dict
@@ -305,19 +303,16 @@ def _fill_from_environmental_variables(self):
         Only parameters which are not already configured are changed (i.e., parameter is None).
 
         """
+        _all_env_dict = gen.load_environment_variables(
+            env_file=self.config.get("env_file", None), env_list=self.config.keys()
+        )
+
         _env_dict = {}
-        try:
-            load_dotenv(self.config["env_file"])
-        except KeyError:
-            pass
         for key, value in self.config.items():
-            # environmental variables for simtools should always start with SIMTOOLS_
-            env_variable_to_read = f"SIMTOOLS_{key.upper()}"
             if value is None:
-                env_value = os.environ.get(env_variable_to_read)
+                env_value = _all_env_dict.get(key)
                 if env_value is not None:
-                    env_value = env_value.split("#")[0].strip().replace('"', "").replace("'", "")
-                _env_dict[key] = env_value
+                    _env_dict[key] = env_value
 
         self._fill_from_config_dict(_env_dict)
 

src/simtools/dependencies.py

@@ -11,11 +11,13 @@
 import logging
 import re
 import subprocess
+from pathlib import Path
 
 import yaml
 
 from simtools import settings
 from simtools.io import ascii_handler
+from simtools.utils import general as gen
 from simtools.version import __version__
 
 _logger = logging.getLogger(__name__)
@@ -192,11 +194,11 @@ def get_build_options(run_time=None):
     """
     Return CORSIKA / sim_telarray config and build options.
 
-    For CORSIKA / sim_telarray build simtools version >0.25.0:
-    expect build_opts.yml  file in each CORSIKA and sim_telarray
+    For CORSIKA / sim_telarray build for simtools version >0.25.0:
+    expects build_opts.yml file in each CORSIKA and sim_telarray
     directories.
 
-    For CORSIKA / sim_telarray build simtools version <=0.25.0:
+    For CORSIKA / sim_telarray build for simtools version <=0.25.0:
     expects a build_opts.yml file in the sim_telarray directory.
 
     Parameters
@@ -211,7 +213,9 @@ def get_build_options(run_time=None):
     """
     build_opts = {}
     for package in ["corsika", "sim_telarray"]:
-        path = getattr(settings.config, f"{package}_path")
+        path = _get_package_path(package)
+        if not path:
+            continue
         try:
             build_opts.update(_get_build_options_from_file(path / "build_opts.yml", run_time))
         except (FileNotFoundError, TypeError, ValueError):
@@ -228,6 +232,14 @@ def get_build_options(run_time=None):
     return build_opts
 
 
+def _get_package_path(package):
+    """Get the package path from settings or environment variables."""
+    path = getattr(settings.config, f"{package}_path")
+    if path is None:
+        path = gen.load_environment_variables().get(f"{package}_path")
+    return Path(path) if path else None
+
+
 def _get_build_options_from_file(build_opts_path, run_time=None):
     """Read build options from file."""
     if run_time is None:
@@ -247,3 +259,21 @@ def _get_build_options_from_file(build_opts_path, run_time=None):
         return yaml.safe_load(result.stdout)
     except yaml.YAMLError as exc:
         raise ValueError(f"Error parsing build_opts.yml from container: {exc}") from exc
+
+
+def export_build_info(output_file, run_time=None):
+    """
+    Export build and version information to a file.
+
+    Parameters
+    ----------
+    output_file : str
+        Path to the output file.
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+    """
+    build_info = get_build_options(run_time)
+    build_info["simtools"] = __version__
+    build_info["database_name"] = get_database_version_or_name(version=False)
+    build_info["database_version"] = get_database_version_or_name(version=True)
+    ascii_handler.write_data_to_file(data=build_info, output_file=Path(output_file))