diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1a9f55f6..f8d2eb87 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -60,6 +60,7 @@ jobs: pytest --cov=. --cov-report html -cov-config=jade.coveragerc | tee pytest_output.log env: ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }} + ACCESS_TOKEN_GITLAB: ${{ secrets.ACCESS_TOKEN_GITLAB }} # Activate environment and run pytest - name: Testing - Windows @@ -68,6 +69,7 @@ jobs: pytest --cov=. --cov-report html -cov-config="jade.coveragerc" | tee pytest_output.log env: ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }} + ACCESS_TOKEN_GITLAB: ${{ secrets.ACCESS_TOKEN_GITLAB }} - name: Archive test results if: always() diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt index 30eebfa2..5bd4b929 100644 --- a/docs/source/requirements.txt +++ b/docs/source/requirements.txt @@ -14,4 +14,5 @@ f4enix >= 0.7.2 sphinx esbonio myst-parser -sphinx_rtd_theme \ No newline at end of file +sphinx_rtd_theme +python-gitlab \ No newline at end of file diff --git a/jade/configuration.py b/jade/configuration.py index 8a212286..a4f5f819 100644 --- a/jade/configuration.py +++ b/jade/configuration.py @@ -104,6 +104,11 @@ def read_settings(self) -> None: self.mpi_exec_prefix = main["Value"].loc["MPI executable prefix"] self.batch_system = main["Value"].loc["Batch system"] self.batch_file = self._process_path(main["Value"].loc["Batch file"]) + # make nea token optional + try: + self.nea_token = main["Value"].loc["NEA token"] + except KeyError: + self.nea_token = None """ Legacy config variables """ # self.xsdir_path = main['Value'].loc['xsdir Path'] @@ -166,7 +171,9 @@ def run_option(self, exp=False) -> str: print( " Cannot submit as a batch job, as no batch system has been defined in the config file." ) - elif (pd.isnull(self.mpi_exec_prefix)) and (pd.isnull(self.mpi_tasks) is not True): + elif (pd.isnull(self.mpi_exec_prefix)) and ( + pd.isnull(self.mpi_tasks) is not True + ): if int(self.mpi_tasks) > 1: print( " Cannot submit batch job as MPI, as no MPI executable prefix has been defined in the config file." diff --git a/jade/constants.py b/jade/constants.py index 599f8c22..0ad268fc 100644 --- a/jade/constants.py +++ b/jade/constants.py @@ -21,3 +21,9 @@ """ CODES = {"MCNP": "mcnp", "Serpent": "serpent", "OpenMC": "openmc", "d1S": "d1s"} + +NEA_TREE = { + "Oktavian_NEA": { + "path": "sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al", + } +} diff --git a/jade/default_settings/Config.xlsx b/jade/default_settings/Config.xlsx index bc7fd2bb..127cffd2 100644 Binary files a/jade/default_settings/Config.xlsx and b/jade/default_settings/Config.xlsx differ diff --git a/jade/input_fetch.py b/jade/input_fetch.py index d8a98846..760217c8 100644 --- a/jade/input_fetch.py +++ b/jade/input_fetch.py @@ -1,21 +1,29 @@ from __future__ import annotations - +from typing import TYPE_CHECKING import tempfile import os import shutil import zipfile import requests +import logging -import jade.main from jade.utilitiesgui import input_with_options +from jade.constants import NEA_TREE +import gitlab + +if TYPE_CHECKING: + import jade.main IAEA_URL = r"https://github.com/IAEA-NDS/open-benchmarks/archive/main.zip" -def fetch_from_git(url: str, authorization_token: str = None) -> str: - """Download a repository from GitHub/GitLab and extract - it to a temporary folder. It can also deal with authentication. +def fetch_from_git( + url: str, authorization_token: str = None, user: str = None, password: str = None +) -> str: + """Download a repository from GitHub and extract + it to a temporary folder. It can also deal with authentication. Supported + authentication is either by token or by username and password. Parameters ---------- @@ -23,6 +31,10 @@ def fetch_from_git(url: str, authorization_token: str = None) -> str: pointer for the zip download authorization_token : str, optional Authorization token to access the IAEA repository. Default is None. + user : str, optional + Username for authentication. Default is None. + password : str, optional + Password for authentication. Default is None. Returns ------- @@ -31,6 +43,8 @@ def fetch_from_git(url: str, authorization_token: str = None) -> str: """ if authorization_token: headers = {"Authorization": f"token {authorization_token}"} + elif user and password: + headers = {"Authorization": f"Basic {user}:{password}"} else: headers = None # Download the repository as a zip file @@ -42,12 +56,20 @@ def fetch_from_git(url: str, authorization_token: str = None) -> str: # Ceck if the download was successful if response.status_code != 200: return False + + return _extract_zip(response.content, os.path.basename(url)) + + +def _extract_zip(binary_zip, dest_name) -> str: # Save the downloaded zip file tmpdirname = tempfile.gettempdir() - tmp_zip = os.path.join(tmpdirname, os.path.basename(url)) + tmp_zip = os.path.join(tmpdirname, dest_name) extracted_folder = os.path.join(tmpdirname, "extracted") + # be sure to clean the folder before extracting + if os.path.exists(extracted_folder): + shutil.rmtree(extracted_folder) with open(tmp_zip, "wb") as f: - f.write(response.content) + f.write(binary_zip) # Extract the zip file with zipfile.ZipFile(tmp_zip, "r") as zip_ref: zip_ref.extractall(extracted_folder) @@ -55,6 +77,50 @@ def fetch_from_git(url: str, authorization_token: str = None) -> str: return extracted_folder +def fetch_from_gitlab( + url: str, path: str, authorization_token: str = None, branch: str = "jade" +) -> str: + """Download a repository from GitLab and extract + it to a temporary folder. It can also deal with authentication. Supported + authentication is by token. + + Parameters + ---------- + url : str + path to the gitlab website (e.g. https://git.oecd-nea.org/) + path : str + path to the repository (e.g. /sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al) + authorization_token : str, optional + Authorization token to access the IAEA repository. Default is None. + branch : str, optional + Branch to download. Default is jade. + + Returns + ------- + extracted_folder: str + path to the extracted folder + """ + gl = gitlab.Gitlab(url=url, private_token=authorization_token) + try: + gl.auth() + except gitlab.exceptions.GitlabAuthenticationError: + logging.error("Gitlab authentication failed") + return False + + # select the correct project + found = False + for project in gl.projects.list(): + if path == project.path_with_namespace: + found = True + break + if not found: + logging.error("Successful authentication but project %s not found" % path) + return False + + binary = project.repository_archive(sha=branch, format="zip") + return _extract_zip(binary, os.path.basename(path) + ".zip") + + def _check_override(session: jade.main.Session, new_inputs: str | os.PathLike) -> bool: """Check if the inputs are already present""" # check which inputs are available and prompt for overwriting @@ -71,6 +137,9 @@ def _check_override(session: jade.main.Session, new_inputs: str | os.PathLike) - def _install_data(fetch_folder: str | os.PathLike, install_folder: str | os.PathLike): for item in os.listdir(fetch_folder): + # if install folder does not exist, create it + if not os.path.exists(install_folder): + os.makedirs(install_folder) # The old folder needs to be deleted first, otherwise the new folder # is saved inside instead of substituting it newpath = os.path.join(install_folder, item) @@ -123,3 +192,50 @@ def fetch_iaea_inputs(session: jade.main.Session) -> bool: _install_data(fetched_folder, install_folder) return True + + +def fetch_nea_inputs(session: jade.main.Session) -> bool: + """Fetch NEA benchmark inputs and experimental data and copy them to + the correct folder in jade structure. In case the inputs + were already present, the user is asked if they want to overwrite them. + + Parameters + ---------- + session : jade.main.Session + JADE session. + + Returns + ------- + bool + True if the inputs were successfully fetched, False otherwise. + """ + # iterate on all the benchmarks + yes_all = False + for key, benchmark in NEA_TREE.items(): + # check if the benchmark is already present + if key in os.listdir(session.path_inputs): + # check for override only if the user did not select yes_all + if not yes_all: + msg = ( + f"{key} is already present. Do you want to overwrite it? [y/n] -> " + ) + ans = input_with_options(msg, ["y", "n", "y_all"]) + if ans == "n": + continue + elif ans == "y_all": + yes_all = True + # fetch the benchmark + path = benchmark["path"] + extracted_folder = fetch_from_gitlab( + "https://git.oecd-nea.org/", + path, + authorization_token=session.conf.nea_token, + ) + # there should only be one folder in the extraction folder + root = os.listdir(extracted_folder)[0] + # all benchmarks should have the same format + exp_data = os.path.join(extracted_folder, root, "01_Experiment_Input", "jade") + inputs = os.path.join(extracted_folder, root, "03_Benchmark_Model", "jade") + # install the data + _install_data(exp_data, os.path.join(session.path_exp_res, key)) + _install_data(inputs, os.path.join(session.path_inputs, key)) diff --git a/setup.cfg b/setup.cfg index 5bf9c6cc..ec982b77 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ install_requires = aspose-words requests f4enix >= 0.7.2 + python-gitlab include_package_data = True diff --git a/tests/input_fetch_test.py b/tests/input_fetch_test.py index c722d149..664e4388 100644 --- a/tests/input_fetch_test.py +++ b/tests/input_fetch_test.py @@ -2,15 +2,17 @@ import os import shutil import pandas as pd +import json + +from jade.input_fetch import fetch_iaea_inputs, fetch_nea_inputs, fetch_from_gitlab +from jade.libmanager import LibManager + cp = os.path.dirname(os.path.abspath(__file__)) # TODO change this using the files and resources support in Python>10 root = os.path.dirname(cp) sys.path.insert(1, root) -from jade.input_fetch import fetch_iaea_inputs -from jade.libmanager import LibManager - ACTIVATION_FILE = os.path.join(cp, "TestFiles", "libmanager", "Activation libs.xlsx") XSDIR_FILE = os.path.join(cp, "TestFiles", "libmanager", "xsdir") @@ -47,6 +49,16 @@ def __init__(self): self.lib = pd.DataFrame( [["00c", "A"], ["31c", "B"]], columns=["Suffix", "name"] ).set_index("Suffix") + token = None + try: + with open( + os.path.join(cp, "secrets.json"), "r", encoding="utf-8" + ) as infile: + token = json.load(infile)["gitlab"] + except FileNotFoundError: + # Then try to get it from GitHub workflow secrets + token = os.getenv("ACCESS_TOKEN_GITLAB") + self.nea_token = token def test_fetch_iaea_inputs(tmpdir, monkeypatch): @@ -93,15 +105,28 @@ def test_fetch_iaea_inputs(tmpdir, monkeypatch): fetch_iaea_inputs(session) assert len(os.listdir(session.path_inputs)) > 1 - # # check failed authentication (this can be used later for gitlab) - # # try to get the token from local secret file - # try: - # with open(os.path.join(cp, "secrets.json"), "r", encoding="utf-8") as infile: - # token = json.load(infile)["github"] - # except FileNotFoundError: - # # Then try to get it from GitHub workflow secrets - # token = os.getenv("ACCESS_TOKEN_GITHUB") - # inputs = iter(["y"]) - # monkeypatch.setattr("builtins.input", lambda msg: next(inputs)) - # ans = fetch_iaea_inputs(session, authorization_token="wrongtoken") - # assert not ans + +def test_fetch_gitlab(): + """Test fetching from GitLab""" + try: + with open(os.path.join(cp, "secrets.json"), "r", encoding="utf-8") as infile: + token = json.load(infile)["gitlab"] + except FileNotFoundError: + # Then try to get it from GitHub workflow secrets + token = os.getenv("ACCESS_TOKEN_GITLAB") + url = "https://git.oecd-nea.org" + path = r"sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al" + extracted_folder = fetch_from_gitlab(url, path, authorization_token=token) + + assert extracted_folder + + +def test_fetch_nea_inputs(tmpdir): + session = SessionMockup( + tmpdir.mkdir("uty"), tmpdir.mkdir("inputs"), tmpdir.mkdir("exp") + ) + + # test correct fetching in an empty folder + fetch_nea_inputs(session) + assert len(os.listdir(session.path_inputs)) > 0 + assert len(os.listdir(session.path_exp_res)) > 0