From 9f35f940ef78f084f58c1acf9f782f7e1a7afb13 Mon Sep 17 00:00:00 2001 From: Davide Laghi Date: Tue, 25 Jun 2024 15:29:25 +0200 Subject: [PATCH 1/5] initial tests --- jade/input_fetch.py | 19 +++++++++++++++---- tests/input_fetch_test.py | 16 +++++++++++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/jade/input_fetch.py b/jade/input_fetch.py index d8a98846..270d5100 100644 --- a/jade/input_fetch.py +++ b/jade/input_fetch.py @@ -1,21 +1,26 @@ from __future__ import annotations - +from typing import TYPE_CHECKING import tempfile import os import shutil import zipfile import requests -import jade.main from jade.utilitiesgui import input_with_options +if TYPE_CHECKING: + import jade.main + IAEA_URL = r"https://github.com/IAEA-NDS/open-benchmarks/archive/main.zip" -def fetch_from_git(url: str, authorization_token: str = None) -> str: +def fetch_from_git( + url: str, authorization_token: str = None, user: str = None, password: str = None +) -> str: """Download a repository from GitHub/GitLab and extract - it to a temporary folder. It can also deal with authentication. + it to a temporary folder. It can also deal with authentication. Supported + authentication is either by token or by username and password. Parameters ---------- @@ -23,6 +28,10 @@ def fetch_from_git(url: str, authorization_token: str = None) -> str: pointer for the zip download authorization_token : str, optional Authorization token to access the IAEA repository. Default is None. + user : str, optional + Username for authentication. Default is None. + password : str, optional + Password for authentication. Default is None. Returns ------- @@ -31,6 +40,8 @@ def fetch_from_git(url: str, authorization_token: str = None) -> str: """ if authorization_token: headers = {"Authorization": f"token {authorization_token}"} + elif user and password: + headers = {"Authorization": f"Basic {user}:{password}"} else: headers = None # Download the repository as a zip file diff --git a/tests/input_fetch_test.py b/tests/input_fetch_test.py index c722d149..b09938bb 100644 --- a/tests/input_fetch_test.py +++ b/tests/input_fetch_test.py @@ -3,14 +3,15 @@ import shutil import pandas as pd +from jade.input_fetch import fetch_iaea_inputs, fetch_from_git +from jade.libmanager import LibManager + + cp = os.path.dirname(os.path.abspath(__file__)) # TODO change this using the files and resources support in Python>10 root = os.path.dirname(cp) sys.path.insert(1, root) -from jade.input_fetch import fetch_iaea_inputs -from jade.libmanager import LibManager - ACTIVATION_FILE = os.path.join(cp, "TestFiles", "libmanager", "Activation libs.xlsx") XSDIR_FILE = os.path.join(cp, "TestFiles", "libmanager", "xsdir") @@ -105,3 +106,12 @@ def test_fetch_iaea_inputs(tmpdir, monkeypatch): # monkeypatch.setattr("builtins.input", lambda msg: next(inputs)) # ans = fetch_iaea_inputs(session, authorization_token="wrongtoken") # assert not ans + + +def test_fetch_gitlab(): + """Test fetching from GitLab""" + url = "https://git.oecd-nea.org/sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al/-/archive/jade/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al-jade.zip" + extracted_folder = fetch_from_git( + url, user="davide.laghi@ext.f4e.europa.eu", password="iXVvEwdo" + ) + assert extracted_folder From ef5d4c1564e8513f97337e48ec8bf1ab70ce0aec Mon Sep 17 00:00:00 2001 From: Davide Laghi Date: Tue, 16 Jul 2024 12:01:39 +0200 Subject: [PATCH 2/5] NEA fetch POC --- jade/configuration.py | 5 +- jade/constants.py | 6 ++ jade/default_settings/Config.xlsx | Bin 14366 -> 14436 bytes jade/input_fetch.py | 111 +++++++++++++++++++++++++++++- tests/input_fetch_test.py | 51 +++++++++----- 5 files changed, 151 insertions(+), 22 deletions(-) diff --git a/jade/configuration.py b/jade/configuration.py index 8a212286..37d559db 100644 --- a/jade/configuration.py +++ b/jade/configuration.py @@ -104,6 +104,7 @@ def read_settings(self) -> None: self.mpi_exec_prefix = main["Value"].loc["MPI executable prefix"] self.batch_system = main["Value"].loc["Batch system"] self.batch_file = self._process_path(main["Value"].loc["Batch file"]) + self.nea_token = main["Value"].loc["NEA token"] """ Legacy config variables """ # self.xsdir_path = main['Value'].loc['xsdir Path'] @@ -166,7 +167,9 @@ def run_option(self, exp=False) -> str: print( " Cannot submit as a batch job, as no batch system has been defined in the config file." ) - elif (pd.isnull(self.mpi_exec_prefix)) and (pd.isnull(self.mpi_tasks) is not True): + elif (pd.isnull(self.mpi_exec_prefix)) and ( + pd.isnull(self.mpi_tasks) is not True + ): if int(self.mpi_tasks) > 1: print( " Cannot submit batch job as MPI, as no MPI executable prefix has been defined in the config file." diff --git a/jade/constants.py b/jade/constants.py index 599f8c22..0ad268fc 100644 --- a/jade/constants.py +++ b/jade/constants.py @@ -21,3 +21,9 @@ """ CODES = {"MCNP": "mcnp", "Serpent": "serpent", "OpenMC": "openmc", "d1S": "d1s"} + +NEA_TREE = { + "Oktavian_NEA": { + "path": "sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al", + } +} diff --git a/jade/default_settings/Config.xlsx b/jade/default_settings/Config.xlsx index bc7fd2bbf6ad887d2a9075d856770a3aa0641e9b..127cffd23d07f8879477aa08df67c99cf3923703 100644 GIT binary patch delta 6734 zcmZ8`by$?$xAqW2D>(=ZB|~=%B^^VDh%`u-(%mE75<|D7fS}aSol?>uC^3`>NQdNy z@A=Mo&-p!nJlAtw>)vbaKc02(d)+(FWy9t3H!R#JQB}PybO2xyfx=>h537<)6(q-O zT93@=PR=!$GUz-O>8|V^6;iLLxDT|&@trNMhDCQv{yuW|PDqFy@f?^WO^t~b@B5t@ zN<3zn^f`bMW!h-DOARSe&LiS>qe9tM99rL>l*OhdCb&PPB?fG;_m$7V%$C?S>=kA* zS?wODg(p%@@}{a9fs&Kq@+B$1;+FE(FTE7>-@OTw?f}Yhl5>{I5vNm*c|KxbW1;$4 znF6Yi89_C6S<6eNg@a;00-{9M)ui+aD6Cz14Yp*HMI%4YJ7HHb-2_~=J|=ioN1CC5 z9M!eL)S;NTFp?n`Z1Y$%p~{;PT*2wmvGcqW<`5 z{$jl5$I7rTyE%G8LQ^R}OV(rgYMIyIUv#ND;Sx%~D2m$xxC%2{4oxCa7zQL7gI`a( zQ2V6{Ey|o1tk5_Kk3(1wlkmR@`($I9d8a#WyHd1fCq#ssd&|tr=pmz0O-D9MrSUmy zKRP8RFX}##Em}RzVMuMPBS>>kRd#z~?Rq#0@GBWkrSfU@ACEkumY#2r zP74|clh5O-r0v&u|~6lq+vyWP7}!M|K5 zUq)p0X}_D}Y*BbPjg8kU_cgIbh&ee5L!mcol7f4L06T6^LPduM_uLG9?#G0d%(16yxq?h%Jfpt>Dg zRdl@787UD~OaS1+Bg6!l63)l%U4qeae7n7N_)Nt#2Ue=nVo^B`f z|I8!V_BC>~He}tQr4G!1-|#L`wR$~z%c5jwq6|K04dG*sjvM_2dGIQ%=6IyG65*aZR`~frxn*+PmUwxVQdpP2z2l z>wCezbAgdU!WFZv^M=!u{iA>ShH+wUzgRzuSPA8wsrA$g{dKi{2xDF7*6wkc!*G)b zqI$Oz7qKzgqY&|^tpGlCp{z1stLqnReGsNT{`s>p@~5@VROitV)mjE7sp+WwZ@S#C z5t6<&Cz(6#$y{yvD8G0x5{_w&WQu3Y{<`q3DtUp}926DaQk-{k=f0RFp`{b5uf^9 zYnM|t{pI#Y_!D@{tyJP8c2hI};Qk))_w_N&z$uiM4AvOZ_V(?V6@xVUuwr8{>+8ab zr+83D?-g4Iaq)r*_Iud)?Aau^nr+a1MYGK7_3p7`0MvmIk3o_8EY1CcwU$?OkV~4(lTsaF?X6jV8(iZ zO}z+8iMvL|J&)_V_fp;I(UzSWN z@I@xrWG=*`=|<+KsMSKb;3WO>s*l5_zV}Y3Lqx>iEVJ4DK(mnMkG$GfaoCG4y_68? zA8lDWZPEqHbB2*Q(G@VhXv^U-a$Jx_wRl5q#TO~X1=_C4pli44%uG#_y29KFZf+9+ ztv9ax8Wd6pr+N63h^^>gy*03pjCF!?VWT zPf1YIv!J9M#>VP$XVR(^{qwJcv4C^D%GSeTkd?dRMOx zDjr45bzI-{3v=AY`=+L ze#Z$npS<+*r9GPsthJn6*D=-cw(c&x`|1k9j_FTH=l!3_$Cgs!Z(O|f zYPEl4OG>E~K~z=Accc_df}}v#&w`|8T@NzqEZNChRYKXhDqLO?e_+spy;s?0kKFl9 zh)mvBW10(AD=!XyIa*W|&B#-#rj`vEIdv?Bt4$GDp)x$iF?9@Iu6xP9SfA9w!Hk zFuc;7Hn8dpSRGcRHFjE@pdL22Xv58MW*+gFsy%Jr_IxxDgp6xE?Op)KS(V@WI0Ay< zNIUt7sN%uGpWL~yiY1Noy;H&kx9&}HoHtV^*J7;)7V4QA&!@$Oos3Su@vEx)^Zvww zD3QjyP!U5J?B#((yw=ZR_0_{C!_=Gi4%5o?))oBJSPE?O-*SEB&!hb;PZdWm0qwE> zTZ9-ST|gVlsY^V!Ufb*E@e;hlmkTeT^?7bIFpcpp<{>ZPzj;3lGMN5y{bnvPYx9us z12%P!?TC+G7u$C;%$@hd8juhL#&}k$9ZuA4@OC0T;kBs4)}Tj7(5hI7wYbzF4H{(V z|0l!1(Fe!;o-*i@3Mim^LVIp0IViNa;#5`e^wMv&08&y&(m{twv0(@7)#J4QLiBiF z1E)o|c2iHK`ok-H?h5ZP5j@~@JolYni?a{ZML_!xb!~t^@TXxwVqHhQN~fh*o!Q<7 zS7x&+nlfbF+lmhVL98AXrCD+^{>OO%CttfcoxBZ`Y(HycAqzRbum|n;uW%vH=Jad7s z2Gi?RPT-sz+W$z7=Dt!QYxm)nn&Zp^ch>!XygbQTobE zx014KlNNWMo#Hn+!hH|)yMf;f+T3p5X_})Bo+QUQqTFlRx6^8NUyA2%A+~T z{n#!Yf7+kP$fv&VjOsyl5Dq(VWKDWBD`7VpqU6KaiSX57D>Z*ugmAeh*CQWfM-B6` zhiSnPopkx?C@U^$JO372lVrgZXA}U)FiO5=m$$ZhCrTY-%16MB4O;6*6s8~jUWZdk z-wHz8^zJIZ1z|QX`-r>1_`4Uw*0ScW#az(KsN`7ISxqRy3y(DF2?j~ak`D@h3@PV)UH9f8QjM z?vRy^IEL4%t>@eT5p)zFIABRc8dC}APw0y;uQeUZtW#ireQvUTBBgB;mIE$av-A?o z_m*hQ9@+g9FqSaC?sRf==YL!ikFs<7;~UT#cAvNI3-xVxDuQM+8$d;GTZ~uWO(z%5 z*1T`wsQ#nVn=|L2j+}Alr$YJMML)J)ZvA<|yr}SnJu8fd`{_+HTs3)Y&%H4-_qELm zf88@T`lK9rJQ6-e3+29~s85jH50tkhUq@wS7AqLlbdq11q{8OXN=C`psU%x!i=U^Z z*^kIFh(?n!S}64;M?r{96{o5>18ttCrLd{C<+10d)>o=z+JVDYJYfdXS`>_625c-} zDh5$l!6lH%j5&e?&P%L{n-OVQ+Cr@{t2dYbN(6N4Rj6FA$ z4J*2vEVOGmaYX4UL_y(WoCsF%;DFxHFXUJ2lV5}mf(AVmv`x4P65rhD3}My`E}@lC zGAB`L8OlIsX+*oI70c!?cIEGkHo{X8I2ro|7|3Nx4 z|3nj2r%E9`FL2#RiI#rsQ16emBeZDL>25}p0)d3bQBcMk!iW*MIX4N+ol}gM4 z>5%v;@Uz^LE%slXe6xs>zdizYO}| z99!vb+gbj~%)g6ofsU5|1RInseZIkmH^5F&W)?8^-Bi<&0TnIQCbd=m42MwMyo^YI z0EtH+Cow5LgBYdg*N5wNivZW(C>+kHW%*F;3q7qK?+?_Z6naD*YJ$=GT+@T|m$FTq z1!tvMPRNlU#OF1$x_G=-I?=Sdr$RLjB^DTmneeQ*N%Mad#mT$#t|ChC`s6YKbztP5 zQkro0QJ})@pG6lSH_?n40m5kiI~zkfHbLF*^IJN=Up580%oL!(5hGw0_~PFIF7=9z zb0zvj{Xq-raxAu2RRrb4uM@>CymPQc)44sMiR@xgzKfw#WgF)Hcu50K6UQ0h;q5djrIt5Y z>s*7{m_ek%>v)f*XD=i5MzmNy*)+_A?uMa0VSSU4?R{tJTXq3@%xA#$rN4^20H+gw z=&-cMwD8zyO+q1ZCFEd#R^a&fhh6&#K2PEtXm8P5V)%*deA|1u?Vz8x-rUHl(7I$S z%9Kq{t+_#4CrTVRiL^}epU-OEL_{`>;Zg?ltbP>xuV$*NT!_iE@*;c*$adeovSL#J zJN+>95m#{dtT7_Ra$y&e=Q5E?jP2C}gAt8A@#TYKV&77_ntg3z9Gdb${m3LMmU=-S z8l7Awlf{(8bEKgl$=9V3=S$xA2W6S^M;`5ZQf+9V>0xw39MOQ_Tf}x&i7G!W!9xvM z0nl=bEM)5eL6yVU!tDuR?dnfAsIfw8QNsw7=cM3Mx~A)XNpes7S6l3|K#qRbV7W$n zFxzOP5x4W#^V62)6qM?0-M7vF%{fYcESJ`DDp9*o(F=swh|~T zWwlU5q)beu86Q~TM?|E8OthAT@B6gX_QbWiR&P#+ji-AHUiHjsezKm9jpp8ex#`wJ z|I1@A$3iwdK?6%?JGqCRy+l8D0F&gk{KtY#-{AzA(U>zgHTw4%eXr9;@Io^{Z>}<= z@5-NXMvB+<(A)I4=1I$boA-1F^xdoDm=3E;PGC*EH%0Dwh& z0D$a4-ugK3xZ9b#SzBp)xY;||y8m@NQgpT47e$DIIy0{c#e#+W42 zW}f*yy3fLHqd`P^RHTHkKv!sNU#KJaX7Xil48~wxXSg}|dncCA!~!UyUMj&+yD1MC z?3m|y;Z~5Gs{i;!7x@woPxN<;FZ`u7T$8Mi&*%myxBWsdsHTITms^$D6&V)}eF&?T zG(1L89Lrh5>jza>9X}N-mA|-@jmtCcPt;u`Z#7xk)KiHid1jFAubxM_c9<6Bb;(ie zxRNAG`Bi#k_%ks1Ly8?1aaBDT^>4PaWV@$N^Ty_+JLkKGv^`dYnBtVs^9b3kjq3_U zj;vlzli`X@B5m6$1Y%cNF;Q|FOyT)16kk8D;ZUSuzz;UIi@&2duW{AQlyD0vNcN;Z z=FpJsvCZqf4mJ}rz@qU*|HOX23JN<`Gcl!>lj ztfFfOfoIdVIJ-@66lH(i6CiLu^zl*eO=V;l9gX5q3FuomD|>p(f!O^1&RykvBU)BA zl@FgPDyetF>CtZjC#0hODLwN&iIcIg{KPsHG+a4FABwe0Z$38@NV=AU9Aq5H zh{9n))p^75MnYnvt)3sS;ILs|j-G~(s$*|%U@~Lpr%n=Qh5Ds>8##YkW^;;oV!BKA zW0Dt_#4WuIBy-yte8j`G@eYbAz4oxln|{|ToO`SXouiFkZRt^o+A|Q+}Pb`hFaejy%-S{Rxk6{!4%`w zP4f#WM3X_YdxcjnOnREd)8kL3KUi-DD>n~$Y&HpP?NITB`P#33O94@^bYtZeb8XII zI_6fg4x~BD@QEEh-tUKd+LO5lLBYm+cVC3wTFMUS2U~`SMu5%q&*VMi@s=T#Jlgeu zZa%UCX|dQMnJ{?UGw7k9~Lm delta 6624 zcmZ8mWmFux*2UeexVx3XwMg;ePN78yclW_{uowWKA z_rCY?BP%QCtevbRXXj*R%PrTfY5@&1`54C(j)Z{F58Xs#1|hLxj%w}86Bhs-zS)K` zbNYR$TBx&hT2QkzMGX6tnComV;b-31oQ~93agaPhN+zO4J#WwwFFXxccQg>bZEmcb zu8sSqzHV3!0Lo6|uo5S3I26tOq@4>V>oRohN=ZOXUQmYV!#q~P*;z}`?tnuH9tjzJ zu3^Ka0TozW8ODeNp1h4A4l;ltOHj>7ae*hI#avOc1FAIsQ*T(;$}!3p=r?{^F5{u7 zV}!aD$g8+KQDKJfI$C)<>(0tjCe@@OLd(@Us}jckAPH_)e8rFWg@h>vg`{+l8;!cB zq@0A8V=qjrS`3Z+q6KED5&-VHp7cxsq7w8B80dDlF0q{gg3L z$gJ8#JZ4S}qjG~{-S9z?rvqS`LGk{q79T^Hh9p*4V2EnQECx!2qVw()h)&tCU#-5W zCwthwprTG>xhZ)`AvgEcYf8LK#?$V;Y(HjP&i6^iXZM8mUdg+QL?rmP+3^bU&2jZT zX&zU(XprXh{S_W{KqA49E|^}gZq$f1k|!nnn#$|!1vHF+r1l8&4HDMp#jf%TWCVmu z9H=Y~J;*p=iH{^|4f_#a?xfnuHYZE;b;;{J8M^4nZR6DMR)jp2%$h{1&w<}RlAe$A z!7^H@K{v|c+0w^#jIQggbzh2%Av#Ek^Qzh2Dxe@UJX-d%u$*`<$NZwe(@XcWvz{Qw z6_^M@7=71oh%iGPs?~eJ{3cB&Y_yK)S$o>k3DCC>t~96R&njFQCHs-~ytq;XSibl@ zv-KpWP6{Pmk;uJan~0D1iVaWlQ=7COcU-3GkuAIvV$=2bj{@0Q-jcj@w4yQNpuXXy za+meN`$_$MqfG7j=?f&I+t~Fmh5iGLD)+4QC#Cn#$j$TmSla|kTtWMmtGin>JUb28s+4}-y}pz$oSspJ zT;62sG-TD*RzR|1MX`Hj-%V;7pQR-h6d69GRb3h0Fwb9eO>YRWM{N?a0LHl-MYM0j z6{q$PNn*))HbG{kna$dP=Wg#T7D{Mo3_)5K8d0u3_o!RVn2mif(H8(Ug?bR$OTIoM zuEG9P&I&o`JP{AvrCMZA;nrzQ=k_pq8W^_)GKtwGEAW>%_QMkD5mLw zHnNR%iq@4j#5PWylxG@Z`P23LvAo-qU{}=CPQKAv zv9S&Yhxib+2FYr`-98hl(a8rkF{yaE3c9p!sUBMhZ~p1rE@%<~QHq#3K8D}M^+YMA z4Ua9>OX|swa*6z7}u}jE3 zFry^&s>R z#J{&f9*R#8ja4{jfXa)CfWScZCnPDLjRa&MOa{h^#Y(OHneu~;MNwpaKF(f^YF6H1=shkf}Q zjd;9VHS54AT!~inGGAO375-1mnfBjglzwx{qh;XwD;;VIs$mMWW6m!SPp>Jhc3^)K zd9p}q*2yDcb~`DGweX9^rw6{?DP4WS7#I7tD>(YqQtv%9u-kZGSg_VSIBlgH1i6|F zo^45{H*k!(ujmZ-Q=0j;LCCGdRTwVHC7JFRr|J$)I}IfIHgL^wEPptQOA#&Qf7)=F zk7^`BoU6*XYEm+D1D6aapb?uN@9gs19OoY^4<}>U=~Jf-h3EJirFZuD?ciEvm)uO8 z6XU`^WWf9lz?=b@;-=bKBF(2D*mHPKJUTV(Sx1uFbKOGnqP&qazO_Pe&L$u9JgxnF zYlqv~bZss_b#}^9{9vL9HE&w`j|k5+{wu&JLm+dG-%;5WRYA63b1#KX|<23RtU-1S#|^odL%Ue+!4%xeVfm`V^8UE|*m|(NOCm-&WD&=Q^z4P66Bg3=Z z_IQbYYHHn@d3^^xd#$L#cp1_-PAvVmh70dcLR`5PhV(2y4G3ij5+wN7fov=nkBq*! zdX6%W%79ptRQlDlie*WYIFa<3%h~np-?2~$xTjMCAw0tRYHU)YtwtUM3BXH1YA9Dw z5gxsHw-7nvnWtK4Du!BUn6g@^dugYZ$kS9EnXx8a<7H}KW{$96bWI!oD$3XkC{Wzq z_-+;6jAkWm1T(oaKN3=`nFh78hRT=-GTO;cf~>T?GG`gLfhBJY?$Di@);+Kzh4ydT z#;+a4W!2%!jZPEj!)ij`y7C>t^y-4$_VVMgMTDd>ooV(wmO`Y8#KPM(Jtn#D6S4OJ zpF(rJks2h2{Q^3P$^hY);}|7`vP=*~J%q3TZhm70qvT<{!wGGkqJFRNfOT1C3Qc@b z5T^r}Ug{s*eAw}C8T>>opzIIAOAV2P^5n+icQMXLqk z(z>QTH64d*cT7|pL1}9Mb2vVmsMP&Y+xhi(s)2nj5Y(gRX*@-2;oxeKHKU5ILzbNUfX$5l7?dV87%iISn5=f%e8HrJI>QN?7#Z zkxc?iK#3KqcM=kGQgz(RHSeR_2$X&;(~0Nxz4C9k_+Z}(5t207q)bGWXh#QLvy9q_ z=M{f4VX0=4Ba4mFS6QpxXEQ*_+&g>Sz1l`e{4>)`HtaOCfSl!XUM9>b0#4jM6dt)p zkYeh0i$o_=1_{A4*)+rn@6mYuyK3e_e39`O0ipBH$CRPJ^!8(aEt$Lh3|a+o7p$rDeAW47Zr#~U`I zK`O6!joi(S`G!+t8nzQFJ^D|T?vfv3_I1VF-BcE0fzYQtX{>A*?=AM<7hpx!lFOnD zL9`j_Eg}+36qAsi^hj!phd@F7Wwstbed)&u?<@0X0U+x@69eTd8^a6#x!RWU_euS_ z)zZI2zTW&;gWh;USfK~IPxT^>vNB{Y{nrXKcRyghTZ}98%b*&edY+yMG21^ z`-f|rMl?y^8H5v-ZMlaE?`33e{YmCCyWD4rHdq@b5(0VnS)7Du>vfL7S<1&8-V-g z4clIU=;j@2BlV@+h-m|Bc311-Eg9V+zQ)SBkpeK9Q&V*m!C)f_cm7SsA|Gh(Oy~z; z^lZ!oS84$OVOAK8#V~yp(-Aq!=-<#3XF`)A#FwETg>i>41II-@EE!XfseWb1@>2b3 zT3Y~M4E-nOR`W;DUyjaee;f-we?)C}B}Z8kAsN(40;lm?TK5jcLJ7^7=CoZM-7^zH5#b9!ZYW_{@-njv9F=xs&V$FAO?oXbq;}MSt{pPPb`Q%f= zIY`0UdjsBkJY!HvU`VR?CX+wGjYhPqe2*Et8Mpayj6h)2X8Ep0DuYgGl>Li37n zcPyC5$ko*}3uA6$&%JZ*LBwe%u@4qUkNMY5zG>po*HVLEZ7`4&tjpJKfui}VD6V9} zg?RGLbonODYN``3FT{s)CEs6=YYk(E)nLIm$k+J7OME7Kr|f(&Iiw7Cx8hSQs>00A zwKU^p?os|36uiCrrH6OGJ^1*)2-dGUeg3nz$-zG}YRw{(m?&k3qgR z(~f84zjzhZMlUBJ9EwlP2Fm$!gP)||(Q&UugdtdH50uY1EVX?r5BW%A6;A5JwB$sj z|9Ck0U=}6t8^1=F)6DXDjr$pU|IS>MK zmt`i!qQgvYiu$}{!1q+Nz~1Dc6uTkioxkQ|S;dxZ{bQY3XvsYbz=JMRq44rgHVA-J z1T3&S?={D?-l!;2EK)V<`t6kPZ9MBUb7V~z{NP367jtOWcn*^!7#b@&d~oe%rT{#s z|3+#roJ>|xJs02#2E~j@V{nb}b-epRsh|t*IP)6qWDSY6JzK_qF(=*nHu{yn(AQ$x zcP~jmLIX)eSU$gp0^7d2g9jZ{##|CZlTKSgaaQ;9q$yN}fvc zWGX8$5nLrP(O3xs>h1U&q+#b9`JVD*Sk&#zN@3&o$0p9&-|p_^s*N+@IS{*%HiOyQHCPBBL0nr7$Mc2%4qcXvN1N`4n)Uv-4aqOjE-jBya)_1aZx=hE+0I4-=TB} zmO^fPpI(1d-}O-njmQ18vuh==W45ntF6x#kI!h}}v{zmw4!|m$2J!>P#S>`)v~@Sd zSQ~QL&D}x*fR+0eT5KB!C398#eeyXzi(4Mu^aq!nA1&pgk~Pruw;|p1oW({7(I`X? z@)h~p{zJ*KBk@-r%JiSJdL43xu_Cfa9q&?Pp320zV23O2a%JQeGQB+!dY?Ht zFCjQJqs3s^PmvAxwF-1%IGB~jqm2E3REsWX3>OVZ-E&ivpv`sWDTX(%2Hgs$MPP)4 zFG~HRJC9k^r@>UBhE7hWxb!Wj-#1GzYko%xR?mF7W;1Ci?;qK;A^sd5eqVF|T}l~C z`)_6Z8FgScOosrxqcBue;#`QSn|O_ep0OFM<5^fw6&dCsRKx!*Igdt(g@PycAeL!+ zZvhizOpy3I)K-`Tf#?C#4j6p{R*un2=)oo_v5>>mnZu1H7Tf2v>F+#w0keLk^M0lM z+D%K$UzgNO&!TeJO$irE=7Nh{cUQXp!(MlT@BIw1{JBKEsZr{~*B~R=m322CJjjV_XnhWX~MX z<^23DIYUx9>cNQeS2dC95*G(iIY~P~A)6r~b8$nT%beG3@nk`$!R)py)K*cnm8Aeg z@q}f+=*w&f6Bw@%bP8kUWXSFc&v=W4ho7daQkO!YUgDU)=4ifUe>N`I2UbSP#pkpz zt(BC%DAJok!8Dm@ux%~JPFQD0DX>yxiOO>$|5>$zL7stpvSnK|zsY@%scx=}Nk-T^ zA7&&bF|hDzO2i?|T;e?%k3Z7)5&U(~b|hUX8g(`S5WF+P*EV=#9ti4p4ssv7QPo}D ztj)`YsnlUHVN^F>o1b1My|wU>K}UuM8w3+>Ti-_*H73xNfXG>bKQg{fAU%wg3LOQM z$frXLlHYc{tMV)L)ZH!o^z%pn$Me+BPrWCRnQ>$!_9a$O@BGy+!#Njpdol?mpo(X$ z$?#<;3stChz#^yv%@@meiFibUAQgluQc1;V2iM4{HnSfaLJDH=$nK7J`7vXLFFkX+ zcJIJD$zS5Oo9%sj@V!<2Xw+mNFBWF&vyltBqe%Ah>IkVm@2wIZXhRtD5m}Vu&yodlBh?yW6-opbIjvuS9D zDau97&R!Tq9;zIMPx^#UT1xzNc0z`L79KMC$qR`ots8Mnnvm=d`JBENIFKl z6%2n!tj5P%&t|!1)nfJA>zUtFI=m)?l|W~Q_wesnVcQYUwZZF|tb|s-yLiexM;NN1F)0P2OWh_U;Zd6a3(~ zmh?F9fLFN=e};wQ7q@-@Q_%~H*e8Xv6i&^v*htbrcp7eFKiFoh{s}l>lzv$U!`wa|!p4Y!BcJS>(1jenWIhr?2 z8i)6z|7Z-SXz`I)RVGMy-H%)b*??A{Q0jT{4YB@^g-km;t!~VC=*(P;9sWy18Pf>0ssI2 diff --git a/jade/input_fetch.py b/jade/input_fetch.py index 270d5100..760217c8 100644 --- a/jade/input_fetch.py +++ b/jade/input_fetch.py @@ -5,8 +5,11 @@ import shutil import zipfile import requests +import logging from jade.utilitiesgui import input_with_options +from jade.constants import NEA_TREE +import gitlab if TYPE_CHECKING: import jade.main @@ -18,7 +21,7 @@ def fetch_from_git( url: str, authorization_token: str = None, user: str = None, password: str = None ) -> str: - """Download a repository from GitHub/GitLab and extract + """Download a repository from GitHub and extract it to a temporary folder. It can also deal with authentication. Supported authentication is either by token or by username and password. @@ -53,12 +56,20 @@ def fetch_from_git( # Ceck if the download was successful if response.status_code != 200: return False + + return _extract_zip(response.content, os.path.basename(url)) + + +def _extract_zip(binary_zip, dest_name) -> str: # Save the downloaded zip file tmpdirname = tempfile.gettempdir() - tmp_zip = os.path.join(tmpdirname, os.path.basename(url)) + tmp_zip = os.path.join(tmpdirname, dest_name) extracted_folder = os.path.join(tmpdirname, "extracted") + # be sure to clean the folder before extracting + if os.path.exists(extracted_folder): + shutil.rmtree(extracted_folder) with open(tmp_zip, "wb") as f: - f.write(response.content) + f.write(binary_zip) # Extract the zip file with zipfile.ZipFile(tmp_zip, "r") as zip_ref: zip_ref.extractall(extracted_folder) @@ -66,6 +77,50 @@ def fetch_from_git( return extracted_folder +def fetch_from_gitlab( + url: str, path: str, authorization_token: str = None, branch: str = "jade" +) -> str: + """Download a repository from GitLab and extract + it to a temporary folder. It can also deal with authentication. Supported + authentication is by token. + + Parameters + ---------- + url : str + path to the gitlab website (e.g. https://git.oecd-nea.org/) + path : str + path to the repository (e.g. /sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al) + authorization_token : str, optional + Authorization token to access the IAEA repository. Default is None. + branch : str, optional + Branch to download. Default is jade. + + Returns + ------- + extracted_folder: str + path to the extracted folder + """ + gl = gitlab.Gitlab(url=url, private_token=authorization_token) + try: + gl.auth() + except gitlab.exceptions.GitlabAuthenticationError: + logging.error("Gitlab authentication failed") + return False + + # select the correct project + found = False + for project in gl.projects.list(): + if path == project.path_with_namespace: + found = True + break + if not found: + logging.error("Successful authentication but project %s not found" % path) + return False + + binary = project.repository_archive(sha=branch, format="zip") + return _extract_zip(binary, os.path.basename(path) + ".zip") + + def _check_override(session: jade.main.Session, new_inputs: str | os.PathLike) -> bool: """Check if the inputs are already present""" # check which inputs are available and prompt for overwriting @@ -82,6 +137,9 @@ def _check_override(session: jade.main.Session, new_inputs: str | os.PathLike) - def _install_data(fetch_folder: str | os.PathLike, install_folder: str | os.PathLike): for item in os.listdir(fetch_folder): + # if install folder does not exist, create it + if not os.path.exists(install_folder): + os.makedirs(install_folder) # The old folder needs to be deleted first, otherwise the new folder # is saved inside instead of substituting it newpath = os.path.join(install_folder, item) @@ -134,3 +192,50 @@ def fetch_iaea_inputs(session: jade.main.Session) -> bool: _install_data(fetched_folder, install_folder) return True + + +def fetch_nea_inputs(session: jade.main.Session) -> bool: + """Fetch NEA benchmark inputs and experimental data and copy them to + the correct folder in jade structure. In case the inputs + were already present, the user is asked if they want to overwrite them. + + Parameters + ---------- + session : jade.main.Session + JADE session. + + Returns + ------- + bool + True if the inputs were successfully fetched, False otherwise. + """ + # iterate on all the benchmarks + yes_all = False + for key, benchmark in NEA_TREE.items(): + # check if the benchmark is already present + if key in os.listdir(session.path_inputs): + # check for override only if the user did not select yes_all + if not yes_all: + msg = ( + f"{key} is already present. Do you want to overwrite it? [y/n] -> " + ) + ans = input_with_options(msg, ["y", "n", "y_all"]) + if ans == "n": + continue + elif ans == "y_all": + yes_all = True + # fetch the benchmark + path = benchmark["path"] + extracted_folder = fetch_from_gitlab( + "https://git.oecd-nea.org/", + path, + authorization_token=session.conf.nea_token, + ) + # there should only be one folder in the extraction folder + root = os.listdir(extracted_folder)[0] + # all benchmarks should have the same format + exp_data = os.path.join(extracted_folder, root, "01_Experiment_Input", "jade") + inputs = os.path.join(extracted_folder, root, "03_Benchmark_Model", "jade") + # install the data + _install_data(exp_data, os.path.join(session.path_exp_res, key)) + _install_data(inputs, os.path.join(session.path_inputs, key)) diff --git a/tests/input_fetch_test.py b/tests/input_fetch_test.py index b09938bb..664e4388 100644 --- a/tests/input_fetch_test.py +++ b/tests/input_fetch_test.py @@ -2,8 +2,9 @@ import os import shutil import pandas as pd +import json -from jade.input_fetch import fetch_iaea_inputs, fetch_from_git +from jade.input_fetch import fetch_iaea_inputs, fetch_nea_inputs, fetch_from_gitlab from jade.libmanager import LibManager @@ -48,6 +49,16 @@ def __init__(self): self.lib = pd.DataFrame( [["00c", "A"], ["31c", "B"]], columns=["Suffix", "name"] ).set_index("Suffix") + token = None + try: + with open( + os.path.join(cp, "secrets.json"), "r", encoding="utf-8" + ) as infile: + token = json.load(infile)["gitlab"] + except FileNotFoundError: + # Then try to get it from GitHub workflow secrets + token = os.getenv("ACCESS_TOKEN_GITLAB") + self.nea_token = token def test_fetch_iaea_inputs(tmpdir, monkeypatch): @@ -94,24 +105,28 @@ def test_fetch_iaea_inputs(tmpdir, monkeypatch): fetch_iaea_inputs(session) assert len(os.listdir(session.path_inputs)) > 1 - # # check failed authentication (this can be used later for gitlab) - # # try to get the token from local secret file - # try: - # with open(os.path.join(cp, "secrets.json"), "r", encoding="utf-8") as infile: - # token = json.load(infile)["github"] - # except FileNotFoundError: - # # Then try to get it from GitHub workflow secrets - # token = os.getenv("ACCESS_TOKEN_GITHUB") - # inputs = iter(["y"]) - # monkeypatch.setattr("builtins.input", lambda msg: next(inputs)) - # ans = fetch_iaea_inputs(session, authorization_token="wrongtoken") - # assert not ans - def test_fetch_gitlab(): """Test fetching from GitLab""" - url = "https://git.oecd-nea.org/sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al/-/archive/jade/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al-jade.zip" - extracted_folder = fetch_from_git( - url, user="davide.laghi@ext.f4e.europa.eu", password="iXVvEwdo" - ) + try: + with open(os.path.join(cp, "secrets.json"), "r", encoding="utf-8") as infile: + token = json.load(infile)["gitlab"] + except FileNotFoundError: + # Then try to get it from GitHub workflow secrets + token = os.getenv("ACCESS_TOKEN_GITLAB") + url = "https://git.oecd-nea.org" + path = r"sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al" + extracted_folder = fetch_from_gitlab(url, path, authorization_token=token) + assert extracted_folder + + +def test_fetch_nea_inputs(tmpdir): + session = SessionMockup( + tmpdir.mkdir("uty"), tmpdir.mkdir("inputs"), tmpdir.mkdir("exp") + ) + + # test correct fetching in an empty folder + fetch_nea_inputs(session) + assert len(os.listdir(session.path_inputs)) > 0 + assert len(os.listdir(session.path_exp_res)) > 0 From 9a95b402981fc93c637d09b097de589279ce248c Mon Sep 17 00:00:00 2001 From: Davide Laghi Date: Tue, 16 Jul 2024 12:05:28 +0200 Subject: [PATCH 3/5] add new python-gitlab dependency --- docs/source/requirements.txt | 3 ++- setup.cfg | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt index 30eebfa2..5bd4b929 100644 --- a/docs/source/requirements.txt +++ b/docs/source/requirements.txt @@ -14,4 +14,5 @@ f4enix >= 0.7.2 sphinx esbonio myst-parser -sphinx_rtd_theme \ No newline at end of file +sphinx_rtd_theme +python-gitlab \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 5bf9c6cc..ec982b77 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ install_requires = aspose-words requests f4enix >= 0.7.2 + python-gitlab include_package_data = True From 674d0abb985498050e193012a57b77ab4d9669b1 Mon Sep 17 00:00:00 2001 From: Davide Laghi Date: Tue, 16 Jul 2024 12:12:52 +0200 Subject: [PATCH 4/5] optional nea token --- jade/configuration.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jade/configuration.py b/jade/configuration.py index 37d559db..a4f5f819 100644 --- a/jade/configuration.py +++ b/jade/configuration.py @@ -104,7 +104,11 @@ def read_settings(self) -> None: self.mpi_exec_prefix = main["Value"].loc["MPI executable prefix"] self.batch_system = main["Value"].loc["Batch system"] self.batch_file = self._process_path(main["Value"].loc["Batch file"]) - self.nea_token = main["Value"].loc["NEA token"] + # make nea token optional + try: + self.nea_token = main["Value"].loc["NEA token"] + except KeyError: + self.nea_token = None """ Legacy config variables """ # self.xsdir_path = main['Value'].loc['xsdir Path'] From d0d856f94672505bc36cc3f39b14ed075f531be6 Mon Sep 17 00:00:00 2001 From: Davide Laghi Date: Tue, 16 Jul 2024 12:23:47 +0200 Subject: [PATCH 5/5] added secret for NEA gitlab --- .github/workflows/pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1a9f55f6..f8d2eb87 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -60,6 +60,7 @@ jobs: pytest --cov=. --cov-report html -cov-config=jade.coveragerc | tee pytest_output.log env: ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }} + ACCESS_TOKEN_GITLAB: ${{ secrets.ACCESS_TOKEN_GITLAB }} # Activate environment and run pytest - name: Testing - Windows @@ -68,6 +69,7 @@ jobs: pytest --cov=. --cov-report html -cov-config="jade.coveragerc" | tee pytest_output.log env: ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }} + ACCESS_TOKEN_GITLAB: ${{ secrets.ACCESS_TOKEN_GITLAB }} - name: Archive test results if: always()