Add NEA SINBAD derived benchmarks

.github/workflows/pytest.yml

@@ -60,6 +60,7 @@ jobs:
           pytest --cov=. --cov-report html -cov-config=jade.coveragerc | tee pytest_output.log
         env:
           ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}
+          ACCESS_TOKEN_GITLAB: ${{ secrets.ACCESS_TOKEN_GITLAB }}
 
       # Activate environment and run pytest
       - name: Testing - Windows
@@ -68,6 +69,7 @@ jobs:
           pytest --cov=. --cov-report html -cov-config="jade.coveragerc" | tee pytest_output.log
         env:
           ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}
+          ACCESS_TOKEN_GITLAB: ${{ secrets.ACCESS_TOKEN_GITLAB }}
 
       - name: Archive test results
         if: always()

docs/source/requirements.txt

@@ -14,4 +14,5 @@ f4enix >= 0.7.2
 sphinx
 esbonio
 myst-parser
-sphinx_rtd_theme
+sphinx_rtd_theme
+python-gitlab

jade/configuration.py

@@ -104,6 +104,11 @@ def read_settings(self) -> None:
         self.mpi_exec_prefix = main["Value"].loc["MPI executable prefix"]
         self.batch_system = main["Value"].loc["Batch system"]
         self.batch_file = self._process_path(main["Value"].loc["Batch file"])
+        # make nea token optional
+        try:
+            self.nea_token = main["Value"].loc["NEA token"]
+        except KeyError:
+            self.nea_token = None
 
         """ Legacy config variables """
         # self.xsdir_path = main['Value'].loc['xsdir Path']
@@ -166,7 +171,9 @@ def run_option(self, exp=False) -> str:
                             print(
                                 " Cannot submit as a batch job, as no batch system has been defined in the config file."
                             )
-                        elif (pd.isnull(self.mpi_exec_prefix)) and (pd.isnull(self.mpi_tasks) is not True):
+                        elif (pd.isnull(self.mpi_exec_prefix)) and (
+                            pd.isnull(self.mpi_tasks) is not True
+                        ):
                             if int(self.mpi_tasks) > 1:
                                 print(
                                     " Cannot submit batch job as MPI, as no MPI executable prefix has been defined in the config file."

jade/constants.py

@@ -21,3 +21,9 @@
 """
 
 CODES = {"MCNP": "mcnp", "Serpent": "serpent", "OpenMC": "openmc", "d1S": "d1s"}
+
+NEA_TREE = {
+    "Oktavian_NEA": {
+        "path": "sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al",
+    }
+}

jade/input_fetch.py

@@ -1,28 +1,40 @@
 from __future__ import annotations
-
+from typing import TYPE_CHECKING
 import tempfile
 import os
 import shutil
 import zipfile
 import requests
+import logging
 
-import jade.main
 from jade.utilitiesgui import input_with_options
+from jade.constants import NEA_TREE
+import gitlab
+
+if TYPE_CHECKING:
+    import jade.main
 
 
 IAEA_URL = r"https://github.com/IAEA-NDS/open-benchmarks/archive/main.zip"
 
 
-def fetch_from_git(url: str, authorization_token: str = None) -> str:
-    """Download a repository from GitHub/GitLab and extract
-    it to a temporary folder. It can also deal with authentication.
+def fetch_from_git(
+    url: str, authorization_token: str = None, user: str = None, password: str = None
+) -> str:
+    """Download a repository from GitHub and extract
+    it to a temporary folder. It can also deal with authentication. Supported
+    authentication is either by token or by username and password.
 
     Parameters
     ----------
     url : str
         pointer for the zip download
     authorization_token : str, optional
         Authorization token to access the IAEA repository. Default is None.
+    user : str, optional
+        Username for authentication. Default is None.
+    password : str, optional
+        Password for authentication. Default is None.
 
     Returns
     -------
@@ -31,6 +43,8 @@ def fetch_from_git(url: str, authorization_token: str = None) -> str:
     """
     if authorization_token:
         headers = {"Authorization": f"token {authorization_token}"}
+    elif user and password:
+        headers = {"Authorization": f"Basic {user}:{password}"}
     else:
         headers = None
     # Download the repository as a zip file
@@ -42,19 +56,71 @@ def fetch_from_git(url: str, authorization_token: str = None) -> str:
     # Ceck if the download was successful
     if response.status_code != 200:
         return False
+
+    return _extract_zip(response.content, os.path.basename(url))
+
+
+def _extract_zip(binary_zip, dest_name) -> str:
     # Save the downloaded zip file
     tmpdirname = tempfile.gettempdir()
-    tmp_zip = os.path.join(tmpdirname, os.path.basename(url))
+    tmp_zip = os.path.join(tmpdirname, dest_name)
     extracted_folder = os.path.join(tmpdirname, "extracted")
+    # be sure to clean the folder before extracting
+    if os.path.exists(extracted_folder):
+        shutil.rmtree(extracted_folder)
     with open(tmp_zip, "wb") as f:
-        f.write(response.content)
+        f.write(binary_zip)
     # Extract the zip file
     with zipfile.ZipFile(tmp_zip, "r") as zip_ref:
         zip_ref.extractall(extracted_folder)
 
     return extracted_folder
 
 
+def fetch_from_gitlab(
+    url: str, path: str, authorization_token: str = None, branch: str = "jade"
+) -> str:
+    """Download a repository from GitLab and extract
+    it to a temporary folder. It can also deal with authentication. Supported
+    authentication is by token.
+
+    Parameters
+    ----------
+    url : str
+        path to the gitlab website (e.g. https://git.oecd-nea.org/)
+    path : str
+        path to the repository (e.g. /sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al)
+    authorization_token : str, optional
+        Authorization token to access the IAEA repository. Default is None.
+    branch : str, optional
+        Branch to download. Default is jade.
+
+    Returns
+    -------
+    extracted_folder: str
+        path to the extracted folder
+    """
+    gl = gitlab.Gitlab(url=url, private_token=authorization_token)
+    try:
+        gl.auth()
+    except gitlab.exceptions.GitlabAuthenticationError:
+        logging.error("Gitlab authentication failed")
+        return False
+
+    # select the correct project
+    found = False
+    for project in gl.projects.list():
+        if path == project.path_with_namespace:
+            found = True
+            break
+    if not found:
+        logging.error("Successful authentication but project %s not found" % path)
+        return False
+
+    binary = project.repository_archive(sha=branch, format="zip")
+    return _extract_zip(binary, os.path.basename(path) + ".zip")
+
+
 def _check_override(session: jade.main.Session, new_inputs: str | os.PathLike) -> bool:
     """Check if the inputs are already present"""
     # check which inputs are available and prompt for overwriting
@@ -71,6 +137,9 @@ def _check_override(session: jade.main.Session, new_inputs: str | os.PathLike) -
 
 def _install_data(fetch_folder: str | os.PathLike, install_folder: str | os.PathLike):
     for item in os.listdir(fetch_folder):
+        # if install folder does not exist, create it
+        if not os.path.exists(install_folder):
+            os.makedirs(install_folder)
         # The old folder needs to be deleted first, otherwise the new folder
         # is saved inside instead of substituting it
         newpath = os.path.join(install_folder, item)
@@ -123,3 +192,50 @@ def fetch_iaea_inputs(session: jade.main.Session) -> bool:
         _install_data(fetched_folder, install_folder)
 
     return True
+
+
+def fetch_nea_inputs(session: jade.main.Session) -> bool:
+    """Fetch NEA benchmark inputs and experimental data and copy them to
+    the correct folder in jade structure. In case the inputs
+    were already present, the user is asked if they want to overwrite them.
+
+    Parameters
+    ----------
+    session : jade.main.Session
+        JADE session.
+
+    Returns
+    -------
+    bool
+        True if the inputs were successfully fetched, False otherwise.
+    """
+    # iterate on all the benchmarks
+    yes_all = False
+    for key, benchmark in NEA_TREE.items():
+        # check if the benchmark is already present
+        if key in os.listdir(session.path_inputs):
+            # check for override only if the user did not select yes_all
+            if not yes_all:
+                msg = (
+                    f"{key} is already present. Do you want to overwrite it? [y/n] -> "
+                )
+                ans = input_with_options(msg, ["y", "n", "y_all"])
+                if ans == "n":
+                    continue
+                elif ans == "y_all":
+                    yes_all = True
+        # fetch the benchmark
+        path = benchmark["path"]
+        extracted_folder = fetch_from_gitlab(
+            "https://git.oecd-nea.org/",
+            path,
+            authorization_token=session.conf.nea_token,
+        )
+        # there should only be one folder in the extraction folder
+        root = os.listdir(extracted_folder)[0]
+        # all benchmarks should have the same format
+        exp_data = os.path.join(extracted_folder, root, "01_Experiment_Input", "jade")
+        inputs = os.path.join(extracted_folder, root, "03_Benchmark_Model", "jade")
+        # install the data
+        _install_data(exp_data, os.path.join(session.path_exp_res, key))
+        _install_data(inputs, os.path.join(session.path_inputs, key))

setup.cfg

@@ -34,6 +34,7 @@ install_requires =
     aspose-words
     requests
     f4enix >= 0.7.2
+    python-gitlab
 
 include_package_data = True
 

tests/input_fetch_test.py

@@ -2,15 +2,17 @@
 import os
 import shutil
 import pandas as pd
+import json
+
+from jade.input_fetch import fetch_iaea_inputs, fetch_nea_inputs, fetch_from_gitlab
+from jade.libmanager import LibManager
+
 
 cp = os.path.dirname(os.path.abspath(__file__))
 # TODO change this using the files and resources support in Python>10
 root = os.path.dirname(cp)
 sys.path.insert(1, root)
 
-from jade.input_fetch import fetch_iaea_inputs
-from jade.libmanager import LibManager
-
 
 ACTIVATION_FILE = os.path.join(cp, "TestFiles", "libmanager", "Activation libs.xlsx")
 XSDIR_FILE = os.path.join(cp, "TestFiles", "libmanager", "xsdir")
@@ -47,6 +49,16 @@ def __init__(self):
         self.lib = pd.DataFrame(
             [["00c", "A"], ["31c", "B"]], columns=["Suffix", "name"]
         ).set_index("Suffix")
+        token = None
+        try:
+            with open(
+                os.path.join(cp, "secrets.json"), "r", encoding="utf-8"
+            ) as infile:
+                token = json.load(infile)["gitlab"]
+        except FileNotFoundError:
+            # Then try to get it from GitHub workflow secrets
+            token = os.getenv("ACCESS_TOKEN_GITLAB")
+        self.nea_token = token
 
 
 def test_fetch_iaea_inputs(tmpdir, monkeypatch):
@@ -93,15 +105,28 @@ def test_fetch_iaea_inputs(tmpdir, monkeypatch):
     fetch_iaea_inputs(session)
     assert len(os.listdir(session.path_inputs)) > 1
 
-    # # check failed authentication (this can be used later for gitlab)
-    # # try to get the token from local secret file
-    # try:
-    #     with open(os.path.join(cp, "secrets.json"), "r", encoding="utf-8") as infile:
-    #         token = json.load(infile)["github"]
-    # except FileNotFoundError:
-    #     # Then try to get it from GitHub workflow secrets
-    #     token = os.getenv("ACCESS_TOKEN_GITHUB")
-    # inputs = iter(["y"])
-    # monkeypatch.setattr("builtins.input", lambda msg: next(inputs))
-    # ans = fetch_iaea_inputs(session, authorization_token="wrongtoken")
-    # assert not ans
+
+def test_fetch_gitlab():
+    """Test fetching from GitLab"""
+    try:
+        with open(os.path.join(cp, "secrets.json"), "r", encoding="utf-8") as infile:
+            token = json.load(infile)["gitlab"]
+    except FileNotFoundError:
+        # Then try to get it from GitHub workflow secrets
+        token = os.getenv("ACCESS_TOKEN_GITLAB")
+    url = "https://git.oecd-nea.org"
+    path = r"sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al"
+    extracted_folder = fetch_from_gitlab(url, path, authorization_token=token)
+
+    assert extracted_folder
+
+
+def test_fetch_nea_inputs(tmpdir):
+    session = SessionMockup(
+        tmpdir.mkdir("uty"), tmpdir.mkdir("inputs"), tmpdir.mkdir("exp")
+    )
+
+    # test correct fetching in an empty folder
+    fetch_nea_inputs(session)
+    assert len(os.listdir(session.path_inputs)) > 0
+    assert len(os.listdir(session.path_exp_res)) > 0