Merge pull request #70 from OpenCOMPES/elab_metadata_integration

Elab metadata integration

Author: rettigl

.cspell/custom-dictionary.txt

@@ -2,6 +2,8 @@
 allclose
 ALLUSERSPROFILE
 amperemeter
+appauthor
+appname
 arange
 archiver
 argwhere
@@ -18,6 +20,7 @@ basepath
 bitshift
 bysource
 calib
+caplog
 checkscan
 clim
 codemirror
@@ -32,16 +35,21 @@ dapolymatrix
 dataconverter
 dataframe
 delaystage
+delenv
 dtype
 dxda
 dxde
 dyda
 dyde
 Ekin
-electronanalyser
+elab
+elabapi
+elabid
+electronanalyzer
 elems
 endstation
 energydispersion
+entityid
 eshift
 faddr
 Faradayweg
@@ -72,6 +80,7 @@ kwds
 labview
 Laurenz
 lensmodes
+levelname
 lineh
 linev
 listf
@@ -102,6 +111,7 @@ Nxpix
 Nxpixels
 Nypixels
 OPCPA
+orcid
 pcolormesh
 Phoibos
 polyfit
@@ -120,6 +130,7 @@ rrvec
 rtol
 rtype
 scanvector
+sharelink
 specsanalyzer
 Specslab
 specsscan
@@ -131,9 +142,11 @@ toctree
 tomlkit
 topfloor
 tqdm
+trarpes
 typehints
 TZCYXS
 undoc
+userid
 venv
 viewcode
 vline

docs/specsanalyzer/config.rst

@@ -4,8 +4,8 @@ The config module contains a mechanics to collect configuration parameters from
 It will load an (optional) provided config file, or alternatively use a passed python dictionary as initial config dictionary, and subsequently look for the following additional config files to load:
 
 * ``folder_config``: A config file of name :file:`specs_config.yaml` in the current working directory. This is mostly intended to pass calibration parameters of the workflow between different notebook instances.
-* ``user_config``: A config file provided by the user, stored as :file:`.specsanalyzer/config.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings.
-* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/specsanalyzer/config.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/specsanalyzer/config.yaml` on Windows. This should provide all necessary default parameters for using the specsanalyzer processor with a given setup. For an example for the setup at the Fritz Haber Institute setup, see :ref:`example_config`
+* ``user_config``: A config file provided by the user, stored as :file:`.config/specsanalyzer/config_v1.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings.
+* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/specsanalyzer/config_v1.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/specsanalyzer/config_v1.yaml` on Windows. This should provide all necessary default parameters for using the specsanalyzer processor with a given setup. For an example for the setup at the Fritz Haber Institute setup, see :ref:`example_config`
 * ``default_config``: The default configuration shipped with the package. Typically, all parameters here should be overwritten by any of the other configuration files.
 
 The config mechanism returns the combined dictionary, and reports the loaded configuration files. In order to disable or overwrite any of the configuration files, they can be also given as optional parameters (path to a file, or python dictionary).

docs/specsscan/helpers.rst

@@ -3,3 +3,7 @@ Helpers
 .. automodule:: specsscan.helpers
    :members:
    :undoc-members:
+
+.. automodule:: specsscan.metadata
+   :members:
+   :undoc-members:

pyproject.toml

@@ -32,15 +32,16 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
+    "elabapi-python>=5.0",
     "h5py>=3.6.0",
     "imutils>=0.5.4",
     "ipympl>=0.9.1",
     "ipywidgets>=7.7.1",
-    "matplotlib>=3.5.1,<3.10.0",
+    "matplotlib>=3.5.1",
     "numpy>=1.21.6",
     "opencv-python>=4.8.1.78",
-    "pynxtools-mpes>=0.2.1",
-    "pynxtools>=0.9.3",
+    "pynxtools-mpes>=0.2.2",
+    "pynxtools>=0.10.1",
     "python-dateutil>=2.8.2",
     "pyyaml>=6.0",
     "xarray>=0.20.2",
@@ -83,12 +84,6 @@ all = [
     "specsanalyzer[dev,docs,notebook]",
 ]
 
-[tool.coverage.report]
-omit = [
-    "config.py",
-    "config-3.py",
-]
-
 [tool.ruff]
 include = ["specsanalyzer/*.py", "specsscan/*.py", "tests/*.py"]
 lint.select = [

src/specsanalyzer/config.py

@@ -1,16 +1,35 @@
 """This module contains a config library for loading yaml/json files into dicts"""
 from __future__ import annotations
 
+import copy
 import json
 import os
 import platform
 from importlib.util import find_spec
 from pathlib import Path
 
 import yaml
+from platformdirs import user_config_path
+
+from specsanalyzer.logging import setup_logging
 
 package_dir = os.path.dirname(find_spec("specsanalyzer").origin)
 
+USER_CONFIG_PATH = user_config_path(
+    appname="specsanalyzer",
+    appauthor="OpenCOMPES",
+    ensure_exists=True,
+)
+SYSTEM_CONFIG_PATH = (
+    Path(os.environ["ALLUSERSPROFILE"]).joinpath("specsanalyzer")
+    if platform.system() == "Windows"
+    else Path("/etc/").joinpath("specsanalyzer")
+)
+ENV_DIR = Path(".env")
+
+# Configure logging
+logger = setup_logging("config")
+
 
 def parse_config(
     config: dict | str = None,
@@ -36,12 +55,13 @@ def parse_config(
         user_config (dict | str, optional): user-based config dictionary
             or file path. The loaded dictionary is completed with the user-based values,
             taking preference over system and default values.
-            Defaults to the file ".specsanalyzer/config.yaml" in the current user's home directory.
+            Defaults to the file ".config/specsanalyzer/config_v1.yaml" in the current user's home
+            directory.
         system_config (dict | str, optional): system-wide config dictionary
             or file path. The loaded dictionary is completed with the system-wide values,
             taking preference over default values.
-            Defaults to the file "/etc/specsanalyzer/config.yaml" on linux,
-            and "%ALLUSERSPROFILE%/specsanalyzer/config.yaml" on windows.
+            Defaults to the file "/etc/specsanalyzer/config_v1.yaml" on linux,
+            and "%ALLUSERSPROFILE%/specsanalyzer/config_v1.yaml" on windows.
         default_config (dict | str, optional): default config dictionary
             or file path. The loaded dictionary is completed with the default values.
             Defaults to *package_dir*/config/default.yaml".
@@ -57,62 +77,51 @@ def parse_config(
         config = {}
 
     if isinstance(config, dict):
-        config_dict = config
+        config_dict = copy.deepcopy(config)
     else:
         config_dict = load_config(config)
         if verbose:
-            print(f"Configuration loaded from: [{str(Path(config).resolve())}]")
+            logger.info(f"Configuration loaded from: [{str(Path(config).resolve())}]")
 
     folder_dict: dict = None
     if isinstance(folder_config, dict):
-        folder_dict = folder_config
+        folder_dict = copy.deepcopy(folder_config)
     else:
         if folder_config is None:
             folder_config = "./specs_config.yaml"
         if Path(folder_config).exists():
             folder_dict = load_config(folder_config)
             if verbose:
-                print(f"Folder config loaded from: [{str(Path(folder_config).resolve())}]")
+                logger.info(f"Folder config loaded from: [{str(Path(folder_config).resolve())}]")
 
     user_dict: dict = None
     if isinstance(user_config, dict):
-        user_dict = user_config
+        user_dict = copy.deepcopy(user_config)
     else:
         if user_config is None:
-            user_config = str(
-                Path.home().joinpath(".specsanalyzer").joinpath("config.yaml"),
-            )
+            user_config = str(USER_CONFIG_PATH.joinpath("config_v1.yaml"))
         if Path(user_config).exists():
             user_dict = load_config(user_config)
             if verbose:
-                print(f"User config loaded from: [{str(Path(user_config).resolve())}]")
+                logger.info(f"User config loaded from: [{str(Path(user_config).resolve())}]")
 
     system_dict: dict = None
     if isinstance(system_config, dict):
-        system_dict = system_config
+        system_dict = copy.deepcopy(system_config)
     else:
         if system_config is None:
-            if platform.system() in ["Linux", "Darwin"]:
-                system_config = str(
-                    Path("/etc/").joinpath("specsanalyzer").joinpath("config.yaml"),
-                )
-            elif platform.system() == "Windows":
-                system_config = str(
-                    Path(os.environ["ALLUSERSPROFILE"])
-                    .joinpath("specsanalyzer")
-                    .joinpath("config.yaml"),
-                )
+            system_config = str(SYSTEM_CONFIG_PATH.joinpath("config_v1.yaml"))
         if Path(system_config).exists():
             system_dict = load_config(system_config)
             if verbose:
-                print(f"System config loaded from: [{str(Path(system_config).resolve())}]")
+                logger.info(f"System config loaded from: [{str(Path(system_config).resolve())}]")
 
     if isinstance(default_config, dict):
-        default_dict = default_config
+        default_dict = copy.deepcopy(default_config)
     else:
         default_dict = load_config(default_config)
         if verbose:
-            print(f"Default config loaded from: [{str(Path(default_config).resolve())}]")
+            logger.info(f"Default config loaded from: [{str(Path(default_config).resolve())}]")
 
     if folder_dict is not None:
         config_dict = complete_dictionary(
@@ -226,3 +235,85 @@ def complete_dictionary(dictionary: dict, base_dictionary: dict) -> dict:
                     dictionary[k] = v
 
     return dictionary
+
+
+def _parse_env_file(file_path: Path) -> dict:
+    """Helper function to parse a .env file into a dictionary.
+
+    Args:
+        file_path (Path): Path to the .env file
+
+    Returns:
+        dict: Dictionary of environment variables from the file
+    """
+    env_content = {}
+    if file_path.exists():
+        with open(file_path) as f:
+            for line in f:
+                line = line.strip()
+                if line and "=" in line:
+                    key, val = line.split("=", 1)
+                    env_content[key.strip()] = val.strip()
+    return env_content
+
+
+def read_env_var(var_name: str) -> str | None:
+    """Read an environment variable from multiple locations in order:
+    1. OS environment variables
+    2. .env file in current directory
+    3. .env file in user config directory
+    4. .env file in system config directory
+
+    Args:
+        var_name (str): Name of the environment variable to read
+
+    Returns:
+        str | None: Value of the environment variable or None if not found
+    """
+    # 1. check OS environment variables
+    value = os.getenv(var_name)
+    if value is not None:
+        logger.debug(f"Found {var_name} in OS environment variables")
+        return value
+
+    # 2. check .env in current directory
+    local_vars = _parse_env_file(ENV_DIR)
+    if var_name in local_vars:
+        logger.debug(f"Found {var_name} in ./.env file")
+        return local_vars[var_name]
+
+    # 3. check .env in user config directory
+    user_vars = _parse_env_file(USER_CONFIG_PATH / ".env")
+    if var_name in user_vars:
+        logger.debug(f"Found {var_name} in user config .env file")
+        return user_vars[var_name]
+
+    # 4. check .env in system config directory
+    system_vars = _parse_env_file(SYSTEM_CONFIG_PATH / ".env")
+    if var_name in system_vars:
+        logger.debug(f"Found {var_name} in system config .env file")
+        return system_vars[var_name]
+
+    logger.debug(f"Environment variable {var_name} not found in any location")
+    return None
+
+
+def save_env_var(var_name: str, value: str) -> None:
+    """Save an environment variable to the .env file in the user config directory.
+    If the file exists, preserves other variables. If not, creates a new file.
+
+    Args:
+        var_name (str): Name of the environment variable to save
+        value (str): Value to save for the environment variable
+    """
+    env_path = USER_CONFIG_PATH / ".env"
+    env_content = _parse_env_file(env_path)
+
+    # Update or add new variable
+    env_content[var_name] = value
+
+    # Write all variables back to file
+    with open(env_path, "w") as f:
+        for key, val in env_content.items():
+            f.write(f"{key}={val}\n")
+    logger.debug(f"Environment variable {var_name} saved to .env file")

src/specsanalyzer/convert.py

@@ -1,9 +1,14 @@
 """Specsanalyzer image conversion module"""
 from __future__ import annotations
 
+import logging
+
 import numpy as np
 from scipy.ndimage import map_coordinates
 
+# Configure logging
+logger = logging.getLogger("specsanalyzer.specsscan")
+
 
 def get_damatrix_from_calib2d(
     lens_mode: str,
@@ -82,7 +87,7 @@ def get_damatrix_from_calib2d(
 
     elif lens_mode in supported_space_modes:
         # use the mode defaults
-        print("This is a spatial mode, using default " + lens_mode + " config")
+        logger.info("This is a spatial mode, using default " + lens_mode + " config")
         rr_vec, da_matrix_full = get_rr_da(lens_mode, calib2d_dict)
         a_inner = da_matrix_full[0][0]
         da_matrix = da_matrix_full[1:][:]