diff --git a/.github/workflows/testpackage_reader.yml b/.github/workflows/testpackage_reader.yml new file mode 100644 index 0000000..ee7d7b1 --- /dev/null +++ b/.github/workflows/testpackage_reader.yml @@ -0,0 +1,47 @@ +name: Testpackage turso + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +env: + CARGO_TERM_COLOR: always + +jobs: + testpackage: + + runs-on: carrington + + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Install python and uv + uses: astral-sh/setup-uv@v6 + - name: Install packages + run: | + module load GCC/11.3.0 + ml Python/3.10.4-GCCcore-11.3.0 + export PY_VERSION="3.10.4" + uv venv -p 3.10.4 CI_env + . CI_env/bin/activate + uv pip install cmake + uv pip install ../analysator-backends + uv pip install git+https://github.com/fmihpc/analysator.git + - name: Import test + run: | + module load GCC/11.3.0 + . CI_env/bin/activate + python -c "import vlsvrs" + python -c "import cpphelpers" + python -c "import analysator" + - name: Run testpackage + run: | + module load GCC/11.3.0 + . CI_env/bin/activate + python ./testpackage/testpackage_vlsvread.py + + + diff --git a/build.rs b/build.rs index ac207a8..8355537 100644 --- a/build.rs +++ b/build.rs @@ -3,7 +3,7 @@ use std::process::Command; use std::{env, path::PathBuf}; fn main() { - // let _cpphelpers = cmake::Config::new("cpphelpers").build(); + let _cpphelpers = cmake::Config::new("cpphelpers").build(); let zfp_dst = cmake::Config::new("rust/external/zfp") .define("BUILD_SHARED_LIBS", "OFF") .build(); diff --git a/cpphelpers/CMakeLists.txt b/cpphelpers/CMakeLists.txt index 3e84101..fe950be 100644 --- a/cpphelpers/CMakeLists.txt +++ b/cpphelpers/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10) project(cpphelpers) # Find Python interpreter + development headers + extension suffix -find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) +find_package(Python3 $ENV{PY_VERSION} EXACT COMPONENTS Interpreter Development.Module REQUIRED) # Build the extension module add_library(cpphelpers SHARED src/cpphelpers.cpp) diff --git a/testpackage/bulk_hermite_compressed.0000001.vlsv b/testpackage/bulk_hermite_compressed.0000001.vlsv new file mode 100644 index 0000000..2d5ce50 Binary files /dev/null and b/testpackage/bulk_hermite_compressed.0000001.vlsv differ diff --git a/testpackage/testpackage_vlsvread.py b/testpackage/testpackage_vlsvread.py new file mode 100644 index 0000000..80c7307 --- /dev/null +++ b/testpackage/testpackage_vlsvread.py @@ -0,0 +1,211 @@ +import analysator as pt +import numpy as np +import vlsvrs +import os +import hashlib +import pickle +import importlib + +datalocation = "/turso/group/spacephysics/analysator/CI/analysator-test-data/vlasiator/" +files=["3D/FID/bulk1/bulk1.0000995.vlsv", + "3D/FHA/bulk1/bulk1.0000990.vlsv", +] +class Tester: + def __init__(self,filename=None): + self.filename=filename + self.vlsvobj=None + self.hashes_dict_rust={} + self.hashes_dict_python={} + def changeFile(self,filename): + self.filename=filename + def loadPickle(self,file): + self.pickled = pickle.load(file) + def dumpPickle(self,file): + pickle.dump(self.hashes_dict,file) + + def load(self,backend=None): + if not backend or backend.lower()=="rust": + self.vlsvobj_rust=vlsvrs.VlsvFile(self.filename) + if not backend or backend.lower()=="python": + self.vlsvobj_python=pt.vlsvfile.VlsvReader(self.filename) + + def setHashTarget(self,backend): + if backend=='rust': + self.vlsvobj=self.vlsvobj_rust + elif backend=='python': + self.vlsvobj=self.vlsvobj_python + else: + print("None set, give valid backend") + + def hash(self,func,args,op=None,opargs=None,both=False,loop=False,flatten=True,sort=False): + + def update(vlsvobj,op,opargs,args,hashdict,loop=False): + #If we want to repeat same function func with different arguments + if loop: + for arg in args: + print(arg,args) + update(vlsvobj,op,opargs,arg,hashdict) + return 0 + argkey=str(args) + opsname="_"+str(op)+"_"+str(opargs) + #Get the method of the vlsvobj that matches the given func str + t=getattr(vlsvobj,func) + #Handle arguments and call the function with the given args to get return value + if type(args) is dict: + retval=t(**args) + elif type(args) is list: + retval=t(*args) + else: + raise IOError(f"Wrong args type: {type(args)} {args}") + #If we want to do operations on the retval for example reshaping, type chaning or sorting + if op and opargs: + #Make into list for handling + if type(op) is not list: + op=[op] + opargs=[opargs] + + for i,f in enumerate(op): + try: + fun=getattr(retval,f) + except AttributeError: + try: + #if given function is not method of retval we make retval the argument of function + if '.' in f: + funcl=f.split('.') + #in case it is inside a module like numpy we need to get instance of the module + funcl[0]=importlib.import_module(funcl[0]) + + fun=getattr(funcl[0],funcl[1]) + else: + fun=f + opargs[i]=[retval] + except AttributeError as e: + raise AttributeError(f"Did not find func {func} to operate with: {e}") + retval=fun(*opargs[i]) + + #save hash of the retval as array + retval=np.array(retval) + print(retval.shape) + if flatten: + retval.reshape((-1,)) + if sort: + retval.sort() + # print(retval) + # print(retval.shape) + # print("end") + if self.filename not in hashdict.keys(): + hashdict[self.filename]={} + if func not in hashdict: + hashdict[self.filename][func]={} + hashdict[self.filename][func][argkey]=[hashlib.sha256(retval.tobytes()).hexdigest(),opsname] + # if vlsvobj==self.vlsvobj_rust: + # if self.filename not in self.hashes_dict_rust.keys(): + # self.hashes_dict_rust[self.filename]={} + # if func not in self.hashes_dict_rust: + # self.hashes_dict_rust[self.filename][func]={} + # self.hashes_dict_rust[self.filename][func][argkey]=[hashlib.sha256(retval.tobytes()).hexdigest(),opsname] + # elif vlsvobj==self.vlsvobj_python: + # if self.filename not in self.hashes_dict_python.keys(): + # self.hashes_dict_python[self.filename]={} + # if func not in self.hashes_dict_python: + # self.hashes_dict_python[self.filename][func]={} + # self.hashes_dict_python[self.filename][func][argkey]=[hashlib.sha256(retval.tobytes()).hexdigest(),opsname] + if not both: + if self.vlsvobj==self.vlsvobj_python: + hashdict=self.hashes_dict_python + elif self.vlsvobj==self.vlsvobj_rust: + hashdict=self.hashes_dict_rust + update(self.vlsvobj,op,opargs,args,hashdict,loop) + else: + update(self.vlsvobj_rust,op,opargs,args,self.hashes_dict_rust,loop) + update(self.vlsvobj_python,op,opargs,args,self.hashes_dict_python,loop) + + def compare(self,funcpy,argspy,funcrust,argsrust): + try: + py=getattr(self.vlsvobj_python,funcpy) + retval_py=py(**argspy) + + rust=getattr(self.vlsvobj_rust,funcrust) + retval_rust=rust(**argsrust) + + except Exception as e: + raise e + + if type(retval_py) is dict and type(retval_rust) is dict : + stack=list(retval_rust.keys()) + if (len(retval_py)!=len(retval_rust)) and len(list(retval_py.keys()))!=0: + raise SystemError("one or both of the dictionaries returned by the readers are empty") + for key in retval_py.keys(): + if retval_rust[key] == retval_py[key]: + stack.remove(key) #maybe a some ohter way to remove it is faster? + else: + raise SystemError("returned dictionary values between vlsvreader and vlsvrs do not match") + if len(stack)!=0: + raise KeyError("returned dictionry from vlsvrs contains keys not present in dictonary returned by python.") + return True + else: + raise NotImplementedError + + +#read ref from file +#(Cellid as variable) reading single cellid ,reading cellid list, (input and output cellid should be same assertion). +#vector (tensor variables from datareduction), read_variable, cellid 0 is nonexistant, error check. +#some datareduction +#read_vdf() (vlsvrs) +# read_variable, compare against fsgird, vg +# read_velocitycells (vlsvreader) + +# read_vdf_spares +# +ciTester = Tester() +# files=["s"] +for file in files: + + #Load data + filename=os.path.join(datalocation,file) + + # filename="/home/siclasse/bulk.0000110.vlsv" + #filename="/home/siclasse/Downloads/bulk_hermite_compressed.0000001.vlsv" + ciTester.changeFile(filename) + ciTester.load() + + #Test compare + cid=ciTester.vlsvobj_python.get_cellid_with_vdf(np.array([0,0,0])) + ciTester.compare("read_velocity_cells",{"cellid":cid,"pop":"proton"},"read_vdf_sparse",{"cid":cid,"pop":"proton"}) + + + #Make hash python + ciTester.setHashTarget("rust") + #ciTester.hash("read_variable",{"variable":"CellID","op":0},op=["reshape","astype","numpy.sort"],opargs=[[tuple([-1])],[int],[]],sort=False,flatten=False) + variables_to_test=["CellID","vg_rhom","vg_v","vg_rhoq","proton/vg_rho","proton/vg_v"] #fg_variable read issue with read_variable + variables_to_test_nonraw=["fg_b","fg_v"] + pylist=ciTester.vlsvobj_python.get_variables() + rustlist=ciTester.vlsvobj_rust.list_variables() + variables=[[var] for var in variables_to_test if (var in pylist and var in rustlist)] + nonraw_vars=[[var,0] for var in variables_to_test_nonraw if (var in pylist and var in rustlist)] + ciTester.hash("read_variable_raw",variables,loop=True) + ciTester.hash("read_variable",nonraw_vars,loop=True) + + #Make hash python + ciTester.setHashTarget("python") + variables.extend([[var[0]] for var in nonraw_vars]) #prob some prettier way than looping through it all but it's not a big list + ciTester.hash("read_variable",variables,loop=True) + + +print(ciTester.hashes_dict_python) +print(ciTester.hashes_dict_rust) +key_map_rust_to_py={"read_variable_raw":"read_variable","read_variable":"read_variable"} +for file in ciTester.hashes_dict_rust.keys(): + print(f"------{file}------") + for key in ciTester.hashes_dict_rust[file].keys(): + py_dict=ciTester.hashes_dict_python[file][key_map_rust_to_py[key]] + rust_dict=ciTester.hashes_dict_rust[file][key] + for argcall in rust_dict.keys(): + if rust_dict[argcall][0]!=py_dict[argcall][0]: + print(rust_dict[argcall][0],py_dict[argcall][0]) + print("Not match") + else: + print("debug", argcall, rust_dict[argcall][0]) + + +