diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..84f7975 --- /dev/null +++ b/.gitignore @@ -0,0 +1,110 @@ +*.o, +*.pyc +*.class +class4gl/__pycache__/* +*.log +.* +build/* +dist/* +trash/* +*/__pychache__/ +*.py[cod] +*$py.class +.Python +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + + + +!/.gitignore diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 0000000..c4510cb --- /dev/null +++ b/MANIFEST @@ -0,0 +1,55 @@ +# file GENERATED by distutils, do NOT edit +setup.cfg +setup.py +class4gl/__init__.py +class4gl/class4gl.py +class4gl/data_global.py +class4gl/data_soundings.py +class4gl/era_advection.py +class4gl/interface_functions.py +class4gl/interface_multi.py +class4gl/model.py +class4gl/interface/interface.py +class4gl/interface/interface_cloudiness.py +class4gl/interface/interface_koeppen.py +class4gl/interface/interface_new_koeppen.py +class4gl/interface/interface_show_profiles.py +class4gl/interface/interface_stations.py +class4gl/interface/taylorDiagram.py +class4gl/interface/test_histogram.py +class4gl/interface/world_histogram.py +class4gl/processing/batch_update_output.py +class4gl/processing/update_output.py +class4gl/ribtol/__init__.py +class4gl/ribtol/ribtol_hw.py +class4gl/ribtol/setup.py +class4gl/setup/batch_setup_era.py +class4gl/setup/batch_setup_global_old.py +class4gl/setup/batch_setup_igra.py +class4gl/setup/batch_update.py +class4gl/setup/batch_update_input.py +class4gl/setup/setup_bllast.py +class4gl/setup/setup_bllast_noon.py +class4gl/setup/setup_era.py +class4gl/setup/setup_global_afternoon.py +class4gl/setup/setup_goamazon.py +class4gl/setup/setup_goamazon_noon.py +class4gl/setup/setup_humppa.py +class4gl/setup/setup_humppa_noon.py +class4gl/setup/setup_igra.py +class4gl/setup/setup_igra_20181217.py +class4gl/setup/setup_igra_pkl.py +class4gl/setup/update_input.py +class4gl/setup/update_setup.py +class4gl/setup/trash/setup_global_old.py +class4gl/simulations/batch_simulations.py +class4gl/simulations/copy_update.py +class4gl/simulations/runmodel.py +class4gl/simulations/simulations.py +class4gl/simulations/simulations_iter.py +class4gl/simulations/simulations_iter_bowen.py +class4gl/simulations/simulations_iter_test.py +class4gl/simulations/simulations_smchange2.py +class4gl/simulations/simulations_veg.py +class4gl/simulations/simulations_wwilt_wfc.py +class4gl/simulations/trash/run_test.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9ba551d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +recursive-include class4gl *.py +recursive-include class4gl *.png +recursive-include class4gl *.pbs +recursive-exclude class4gl *.pyc +# include simulations/* +# include setup/* +# include processing/* + + diff --git a/README.md b/README.md new file mode 100644 index 0000000..2eb73e1 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +[![CLASS4GL Logo](https://class4gl.eu/wp-content/uploads/2019/01/cropped-class4gl_small-1.png)](https://class4gl.eu) + +_CLASS4GL_ (Chemistry Land-surface Atmosphere Soil Slab model for Global Studies) is a fast and easy interface to investigate the dynamics of the atmospheric boundary layer from weather balloons worldwide. General info and tutorials for using CLASS4GL are available at https://class4gl.eu, and video clips about the atmospheric boundary layer physics can be found on the [website of the original CLASS model](classmodel.github.io/). + +# Features + - _Mine_ appropriate observations from global radio soundings, satellite data, reanalysis and climate models + - _Automise_ mass parallel simulations of the atmospheric boundary layer and global sensitivity experiments + - _Foster_ a better understanding of land-atmosphere interactions and the drivers of extreme weather globally + - _Share_ your data, experiments, and code developments with the research community + +# Method + +### Description + +The framework CLASS4GL is designed to facilitate the investigation of the atmospheric boundary layer evolution in response to different land and atmospheric conditions observed around the world. The core of the platform is the model CLASS that is used to simulate the evolution of the atmospheric boundary layer. Instruction video about the boundary layer processes and how they are considered in the CLASS model can be found as on the [CLASS model website](https://classmodel.github.io/). Observational data from balloons, satellites and reanalysis, are used to constrain and initialize the model. CLASS4GL uses 2 million global balloon soundings from the integrated global radio sounding archive and satellite data from the last 40 years. + +### Components + + - A global data module that employs balloon soundings, satellite imagery and reanalysis data + - An interface to easily perform multiple simulations of the atmospheric boundary layer in parallel, and multiple batches of global sensitivity experiments + - Tools for Pre-and post-processing the data pool of input data and experiments. + - A GUI data explorer + +The tool is under continuous development, and it can downloaded and installed as described in the tutorials on class4gl.eu/#getstarted. + +In case you experience a problem or a bug, please don’t hesitate to contact us class4gl.eu/#contact. You an also open an issue on the github page (https://github.com/hendrikwout/class4gl/issues) . Any feedback will be highly appreciated. + +### Data sources + +CLASS4GL employs the balloon soundings from the Integrated Global Radiosonde Archive (IGRA) to initialize and validate the CLASS model. The sounding data is supplemented with ancillary data to further constrain the model. Therefore, a default set of gridded global datasets from satellite imagery, reanalysis and and surveys have been used that span a period of 1981–2015. An complete overview of the datasets can be found in the table. However, the default set can be replaced by alternative datasets as long as they are provided in netCDF format. + +[Schematic overview of CLASS4GL](https://class4gl.eu//wp-content/uploads/2019/01/image4-1024x794.png) + +A CLASS4GL data package is available that can be directly used to perform and validate ABL model simulations and sensitivity experiments. The locations of the balloon soundings are performed for different climate regions as shown on the map. + +[150 stations from IGRA of the reference dataset to perform and validate the ABL model simulations with CLASS4GL (see Sect. 2.2 of the CLASS4GL manuscript). The different climate classes are indicated with the colors according to the Köppen-Geiger climate classification. The markers indicate the locations of the atmospheric profiles from three observation campaigns (ie., HUMPPA, BLLAST and GOAMAZON)](https://class4gl.eu/wp-content/uploads/2019/01/image-1-480x300.png)] + +[Data library of CLASS4GL](https://class4gl.eu/wp-content/uploads/2019/01/image-5-768x492.png) + +### Reference +H. Wouters, I. Y. Petrova, C. C. van Heerwaarden, J. Vilà-Guerau de Arellano, A. J. Teuling, J. A. Santanello, V. Meulenberg, D. G. Miralles. A novel framework to investigate atmospheric boundary layer dynamics from balloon soundings worldwide: CLASS4GL v1.0. In preparation. + + +# Get started: +see https://class4gl.eu/#getstarted + + diff --git a/class4gl/Equirectangular_projection_SW.png b/class4gl/Equirectangular_projection_SW.png new file mode 100644 index 0000000..047817a Binary files /dev/null and b/class4gl/Equirectangular_projection_SW.png differ diff --git a/class4gl/__init__.py b/class4gl/__init__.py new file mode 100644 index 0000000..d192b95 --- /dev/null +++ b/class4gl/__init__.py @@ -0,0 +1,11 @@ +# from . import model,class4gl,interface_multi,data_air,data_global +from ribtol import * +from setup import * +from simulations import * +from processing import * + +__version__ = '0.1.0' + +__author__ = 'Hendrik Wouters ' + +__all__ = [] diff --git a/class4gl/class4gl.py b/class4gl/class4gl.py new file mode 100644 index 0000000..f54db96 --- /dev/null +++ b/class4gl/class4gl.py @@ -0,0 +1,2011 @@ +# -*- coding: utf-8 -*- + +""" + +Created on Mon Jan 29 12:33:51 2018 + +Module file for class4gl, which extents the class-model to be able to take +global air profiles as input. It exists of: + +CLASSES: + - an input object, namely class4gl_input. It includes: + - a function to read Wyoming sounding data from a yyoming stream object + - a function to read global data from a globaldata library object + - the model object: class4gl + - .... + +DEPENDENCIES: + - xarray + - numpy + - data_global + - Pysolar + - yaml + +@author: Hendrik Wouters + +""" + + + +""" Setup of envirnoment """ + +# Standard modules of the stand class-boundary-layer model +from model import model +from model import model_output as class4gl_output +from model import model_input +from model import qsat +#from data_soundings import wyoming + +import importlib +spam_loader = importlib.find_loader('Pysolar') +found = spam_loader is not None +if found: + import Pysolar + import Pysolar.util as Pysolarutil + GetSunriseSunset = Pysolarutil.GetSunriseSunset + GetAzimuth = Pysolarutil.solar.GetAzimuth + GetAltitude = Pysolarutil.solar.GetAltitude +else: + import pysolar as Pysolar + Pysolarutil = Pysolar.util + GetSunriseSunset = Pysolarutil.get_sunrise_sunset + GetAzimuth = Pysolar.solar.get_azimuth + GetAltitude = Pysolar.solar.get_altitude +import yaml +import logging +import warnings +import pytz + +#formatter = logging.Formatter() +logging.basicConfig(format='%(asctime)s - \ + %(name)s - \ + %(levelname)s - \ + %(message)s') + + +# Generic Python Packages +import numpy as np +import datetime as dt +import pandas as pd +import xarray as xr +import io +#from skewt.thermodynamics import TempK,DewPoint,MixR2VaporPress,GammaW,degCtoK, Rs_da, Cp_da,VaporPressure,MixRatio +from data_global import data_global +grav = 9.81 + +# this is just a generic input object +class generic_input(object): + def __init__(self): + self.init = True + + +# all units from all variables in CLASS(4GL) should be defined here! +units = { + 'h':'m', + 'theta':'K', + 'q':'kg/kg', + 'cc': '-', + 'cveg': '-', + 'wg': 'm3 m-3', + 'w2': 'm3 m-3', + #'wg': 'kg/kg', + 'Tsoil': 'K', + 'T2': 'K', + 'z0m': 'm', + 'alpha': '-', + 'LAI': '-', + 'dhdt':'m/h', + 'dthetadt':'K/h', + 'dqdt':'kg/kg/h', + 'BR': '-', + 'EF': '-', + 'advt_x': 'K/s', + 'advt_y': 'K/s', +} + +class class4gl_input(object): + """ + this is the class4gl_input. It extends the model_input, which is now + assigned to self.pars. It now also includes initial profiles as pandas + Dataframes: + self.air_balloon: raw profile input for profile of u,v,theta,q (not used) + self.air_ap : the same as self.air_balloonm, but for which a mixed + layer is fitted. Thi profile is used as input. + self.air_ac : atmospheric circulation profiles for advection and + subsidence + + # FYI this was the way it was defined in an early version: + # class4gl_input = type('class4gl_input', (model_input,gl_input,gl_dia), dict(c='c')) + """ + + def __init__(self,set_pars_defaults=True,debug_level=logging.WARNING): + + """ set up logger (see: https://docs.python.org/2/howto/logging.html) + """ + + self.logger = logging.getLogger('class4gl_input') + if debug_level is not None: + self.logger.setLevel(debug_level) + + # # create logger + # #self.logger = logging.getLogger('class4gl_input') + # #self.logger.setLevel(debug_level) + + # # create console handler and set level to debug + # ch = logging.StreamHandler() + # ch.setLevel(debug_level) + + # # create formatter + # formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + # # add formatter to ch + # ch.setFormatter(formatter) + + # # add ch to logger + # self.logger.addHandler(ch) + # # print("TESTTESTSETSTETETS") + # # self.logger.warning("testsetsetsttets") + # #stop + + # # """ end set up logger """ + + + + # these are the standard model input single-value parameters for class + self.pars = model_input() + + # diagnostic parameters of the initial profile + self.diag = dict() + + # In this variable, we keep track of the different parameters from where it originates from. + self.sources = {} + + if set_pars_defaults: + self.set_pars_defaults() + + def set_pars_defaults(self): + + """ + Create empty model_input and set up case + """ + defaults = dict( + dt = 60. , # time step [s] + runtime = 6*3600 , # total run time [s] + + # mixed-layer input + sw_ml = True , # mixed-layer model switch + sw_shearwe = False , # shear growth mixed-layer switch + sw_fixft = False , # Fix the free-troposphere switch + h = 200. , # initial ABL height [m] + Ps = 101300., # surface pressure [Pa] + divU = 0. , # horizontal large-scale divergence of wind [s-1] + #fc = 1.e-4 , # Coriolis parameter [m s-1] + + theta = 288. , # initial mixed-layer potential temperature [K] + dtheta = 1. , # initial temperature jump at h [K] + gammatheta = 0.006 , # free atmosphere potential temperature lapse rate [K m-1] + gammatheta_lower_limit = 0.002, + advtheta = 0. , # advection of heat [K s-1] + beta = 0.2 , # entrainment ratio for virtual heat [-] + wtheta = 0.1 , # surface kinematic heat flux [K m s-1] + + q = 0.008 , # initial mixed-layer specific humidity [kg kg-1] + dq = -0.001 , # initial specific humidity jump at h [kg kg-1] + gammaq = 0. , # free atmosphere specific humidity lapse rate [kg kg-1 m-1] + advq = 0. , # advection of moisture [kg kg-1 s-1] + wq = 0.1e-3 , # surface kinematic moisture flux [kg kg-1 m s-1] + + CO2 = 422. , # initial mixed-layer CO2 [ppm] + dCO2 = -44. , # initial CO2 jump at h [ppm] + gammaCO2 = 0. , # free atmosphere CO2 lapse rate [ppm m-1] + advCO2 = 0. , # advection of CO2 [ppm s-1] + wCO2 = 0. , # surface kinematic CO2 flux [ppm m s-1] + sw_wind = True , # prognostic wind switch + u = 0. , # initial mixed-layer u-wind speed [m s-1] + du = 0. , # initial u-wind jump at h [m s-1] + gammau = 0. , # free atmosphere u-wind speed lapse rate [s-1] + advu = 0. , # advection of u-wind [m s-2] + v = 0.0 , # initial mixed-layer u-wind speed [m s-1] + dv = 0.0 , # initial u-wind jump at h [m s-1] + gammav = 0. , # free atmosphere v-wind speed lapse rate [s-1] + advv = 0. , # advection of v-wind [m s-2] + sw_sl = True , # surface layer switch + ustar = 0.3 , # surface friction velocity [m s-1] + z0m = 0.02 , # roughness length for momentum [m] + z0h = 0.02* 0.1 , # roughness length for scalars [m] + sw_rad = True , # radiation switch + lat = 51.97 , # latitude [deg] + lon = -4.93 , # longitude [deg] + doy = 268. , # day of the year [-] + tstart = 6.8 , # time of the day [h UTC] + cc = 0.0 , # cloud cover fraction [-] + Q = 400. , # net radiation [W m-2] + dFz = 0. , # cloud top radiative divergence [W m-2] + ls_type = 'js' , # land-surface parameterization ('js' for Jarvis-Stewart or 'ags' for A-Gs) + wg = 0.21 , # volumetric water content top soil layer [m3 m-3] + w2 = 0.21 , # volumetric water content deeper soil layer [m3 m-3] + cveg = 0.85 , # vegetation fraction [-] + Tsoil = 295. , # temperature top soil layer [K] + Ts = 295. , # initial surface temperature [K] + T2 = 296. , # temperature deeper soil layer [K] + a = 0.219 , # Clapp and Hornberger retention curve parameter a + b = 4.90 , # Clapp and Hornberger retention curve parameter b + p = 4. , # Clapp and Hornberger retention curve parameter c + CGsat = 3.56e-6, # saturated soil conductivity for heat + wsat = 0.472 , # saturated volumetric water content ECMWF config [-] + wfc = 0.323 , # volumetric water content field capacity [-] + wwilt = 0.171 , # volumetric water content wilting point [-] + C1sat = 0.132 , + C2ref = 1.8 , + LAI = 2. , # leaf area index [-] + gD = 0.0 , # correction factor transpiration for VPD [-] + rsmin = 110. , # minimum resistance transpiration [s m-1] + rssoilmin = 50. , # minimun resistance soil evaporation [s m-1] + alpha = 0.25 , # surface albedo [-] + Wmax = 0.0012 , # thickness of water layer on wet vegetation [m] + Wl = 0.0000 , # equivalent water layer depth for wet vegetation [m] + Lambda = 5.9 , # thermal diffusivity skin layer [-] + c3c4 = 'c3' , # Plant type ('c3' or 'c4') + sw_cu = False , # Cumulus parameterization switch + dz_h = 150. , # Transition layer thickness [m] + cala = None , # soil heat conductivity [W/(K*m)] + crhoc = None , # soil heat capacity [J/K*m**3] + sw_ls = True , + sw_ap = True , # switch that tells to initialize with fitted Air Profiles (eg., from balloon soundings) as input + sw_ac = None , # switch that tells to use large-scale gridded Air Circulation (advection and subsindence) fields as input from eg., ERA-INTERIM + sw_lit = False, + ) + pars = model_input() + for key in defaults: + pars.__dict__[key] = defaults[key] + + self.update(source='defaults',pars=pars) + + def clear(self): + """ this procudure clears the class4gl_input """ + + for key in list(self.__dict__.keys()): + del(self.__dict__[key]) + self.__init__() + + def dump(self,file): + """ this procedure dumps the class4gl_input object into a yaml file + + Input: + - self.__dict__ (internal): the dictionary from which we read + Output: + - file: All the parameters in self.__init__() are written to + the yaml file, including pars, air_ap, sources etc. + """ + file.write('---\n') + index = file.tell() + file.write('# CLASS4GL input; format version: 0.1\n') + + # write out the position of the current record + yaml.dump({'index':index}, file, default_flow_style=False) + + # we do not include the none values + for key,data in self.__dict__.items(): + #if ((type(data) == model_input) or (type(class4gl_input): + if key == 'pars': + + pars = {'pars' : self.__dict__['pars'].__dict__} + parsout = {} + for key in pars.keys(): + if pars[key] is not None: + parsout[key] = pars[key] + + yaml.dump(parsout, file, default_flow_style=False) + elif type(data) == dict: + if key == 'sources': + # in case of sources, we want to have a + # condensed list format as well, so we leave out + # 'default_flow_style=False' + yaml.dump({key : data}, file) + else: + yaml.dump({key : data}, file, + default_flow_style=False) + elif type(data) == pd.DataFrame: + # in case of dataframes (for profiles), we want to have a + # condensed list format as well, so we leave out + # 'default_flow_style=False' + yaml.dump({key: data.to_dict(orient='list')},file) + + # # these are trials to get it into a more human-readable + # fixed-width format, but it is too complex + #stream = yaml.dump({key : False},width=100, default_flow_style=False) + #file.write(stream) + + # workaround. I don't know how to put a table in a readable format by using yaml. So I do it manually here + #file.write(key+': !!str |\n') + #file.write(str(data)+'\n') + + def load_yaml_dict(self,yaml_dict,reset=True): + """ this procedure loads class4gl_input data from a dictionary obtained from yaml + + Input: + - yaml_dict: the dictionary from which we read + - reset: reset data before reading + Output: + - All the parameters in self, eg., (pars, air_ap, sources etc.,). + """ + + if reset: + for key in list(self.__dict__.keys()): + del(self.__dict__[key]) + self.__init__() + + for key,data in yaml_dict.items(): + if key == 'pars': + self.__dict__[key] = model_input() + self.__dict__[key].__dict__ = data + elif key in ['air_ap','air_balloon','air_ac','air_ach']: + self.__dict__[key] = pd.DataFrame(data) + elif key == 'sources': + self.__dict__[key] = data + elif key == 'diag': + self.__dict__[key] = data + else: + warnings.warn("Key '"+key+"' may not be implemented.") + self.__dict__[key] = data + + def update(self,source,**kwargs): + """ this procedure is to make updates of input parameters and tracking + of their source more convenient. It implements the assignment of + parameter source/sensitivity experiment IDs ('eg., + 'defaults', 'sounding balloon', any satellite information, climate + models, sensitivity tests etc.). These are all stored in a convenient + way with as class4gl_input.sources. This way, the user can always consult with + from where parameters data originates from. + + Input: + - source: name of the underlying dataset + - **kwargs: a dictionary of data input, for which the key values + refer to the class4gl data type ('pars', 'air_ap', 'air_balloon', etc.) and + the values is a again a dictionary/dataframe of datakeys/columns + ('wg','PRES','datetime', ...) and datavalues (either single values, + profiles ...), eg., + + pars = {'wg': 0.007 , 'w2', 0.005} + pars = {pd.Dataframe('PRES': [1005.,9523,...] , 'THTA': [295., + 300.,...]} + + Output: + - self.__dict__[datatype] : object to which the parameters are + assigned. They can be consulted with + self.pars, self.profiles, etc. + + - self.sources[source] : It supplements the overview overview of + data sources can be consulted with + self.sources. The structure is as follows: + as: + self.sources = { + 'wyoming': ['pars:datetime','air_balloon:PRES','air_ap:QABS', ...], + 'GLEAM' : ['pars:wg','pars:w2', ...], + ... + } + + """ + + #print(source,kwargs) + + for key,data in kwargs.items(): + #print('update',key,data) + + #print(key) + # if the key is not in class4gl_input object, then just add it. In + # that case, the update procedures below will just overwrite it + if key not in self.__dict__: + self.__dict__[key] = data + + + + + #... we do an additional check to see whether there is a type + # match. I not then raise a key error + if (type(data) != type(self.__dict__[key]) \ + # we allow dict input for model_input pars + and not ((key == 'pars') and (type(data) == dict) and \ + (type(self.__dict__[key]) == model_input))): + + raise TypeError('input key '+key+' is not of the same type as the one in the class4gl_object') + + + # This variable keeps track of the added data that is supplemented + # by the current source. We add this to class4gl_input.sources + datakeys = [] + + #... and we update the class4gl_input data, and this depends on the + # data type + + if type(self.__dict__[key]) == pd.DataFrame: + # If the data type is a dataframe, then we update the columns + for column in list(data.columns): + #print(column) + self.__dict__[key][column] = data[column] + datakeys.append(column) + + + elif type(self.__dict__[key]) == model_input: + # if the data type is a model_input (pars), then we update its internal + # dictionary of parameters + if type(data) == model_input: + self.__dict__[key].__dict__ = {**self.__dict__[key].__dict__, \ + **data.__dict__} + datakeys = list(data.__dict__.keys()) + elif type(data) == dict: + datakeys = list(data.keys()) + datavalues = list(data.values()) + for idatavalue,datavalue in enumerate(datavalues): + + # convert numpy to native python value types, so that + # we get clean output in the yaml file + if type(datavalue).__module__ == 'numpy': + datavalues[idatavalue] = datavalue.item() + + self.__dict__[key].__dict__ = \ + {**self.__dict__[key].__dict__, \ + **dict(zip(datakeys, datavalues))} + else: + raise TypeError('input key '+key+' is not of the same type\ + as the one in the class4gl_object') + + + + elif type(self.__dict__[key]) == dict: + # if the data type is a dictionary, we update the + # dictionary + # print('before update', self.__dict__[key] , data) + self.__dict__[key] = {self.__dict__[key] , data} + # print('after update',self.__dict__[key] ) + datakeys = list(data.keys()) + + + # if source entry is not existing yet, we add it + if source not in self.sources.keys(): + self.sources[source] = [] + + + # self.logger.debug('updating section "'+\ + # key+' ('+' '.join(datakeys)+')'\ + # '" from source \ + # "'+source+'"') + + # Update the source dictionary: add the provided data keys to the + # specified source list + for datakey in datakeys: + # At first, remove the occurences of the keys in the other + # source lists + for sourcekey,sourcelist in self.sources.items(): + if key+':'+datakey in sourcelist: + self.sources[sourcekey].remove(key+':'+datakey) + # Afterwards, add it to the current source list + self.sources[source].append(key+':'+datakey) + + + # # in case the datatype is a class4gl_input_pars, we update its keys + # # according to **kwargs dictionary + # if type(self.__dict__[datatype]) == class4gl_input_pars: + # # add the data parameters to the datatype object dictionary of the + # # datatype + # self.__dict__[datatype].__dict__ = {**self.__dict__[datatype].__dict__ , + # **kwargs} + # # in case, the datatype reflects a dataframe, we update the columns according + # # to the *args list + # elif type(self.__dict__[datatype]) == pd.DataFrame: + # for dataframe in args: + # for column in list(dataframe.columns): + # self.__dict__[datatype][column] = dataframe[column] + + + def get_profile(self,IOBJ, *args, **argv): + # if type(IOBJ) == wyoming: + self.get_profile_wyoming(IOBJ,*args,**argv) + # else: + # raise TypeError('Type '+str(type(IOBJ))+' is not supported') + + def get_profile_wyoming(self,wy_strm,air_ap_mode = 'b'): + """ + Purpose: + This procedure assigns wyoming air profiles and parameters to the class4gl_input object. + + Input: + 1. wy_strm = wyoming html (beautifulsoup) stream object. The + function will take the profile at the stream's current + position. + 2. air_ap_mode: which air profile do we take? + - b : best + - l : according to lower limit for the mixed-layer height + estimate + - u : according to upper limit for the mixed-layer height + estimate + + + Output: + 1. all single-value parameters are stored in the + class4gl_input.pars object + 2. the souding profiles are stored in the in the + class4gl_input.air_balloon dataframe + 3. modified sounding profiles for which the mixed layer height + is fitted + 4. ... + + """ + + + # Raise an error in case the input stream is not the correct object + # if type(wy_strm) is not wyoming: + # raise TypeError('Not a wyoming type input stream') + + # Let's tell the class_input object that it is a Wyoming fit type + self.air_ap_type = 'wyoming' + # ... and which mode of fitting we apply + self.air_ap_mode = air_ap_mode + + """ Temporary variables used for output """ + # single value parameters derived from the sounding profile + dpars = dict() + # profile values + air_balloon = pd.DataFrame() + # fitted profile values + air_ap = pd.DataFrame() + + string = wy_strm.current.find_next('pre').text + string = string.split('\n')[:-1] + string = '\n'.join(string) + + columns = [ 'PRES', 'HGHT', 'TEMP', 'DWPT', 'RELH', 'MIXR', 'DRCT','SKNT' , 'THTA','THTE', 'THTV'] + air_balloon_in = pd.read_fwf(io.StringIO(str(string)),widths=[7]*11,names=columns,skiprows=5,dtype=np.float,skipfooter=0)#.iloc[5:-1] + #ONE_COLUMN = pd.read_table(io.StringIO(str(string)),sep=r"\s*",skiprows=[0,1,3,4]) + + #string = soup.pre.next_sibling.next_sibling + + string = wy_strm.current.find_next('pre').find_next('pre').text + + # this crazy long line just loads the sounding parameter table into parameters object (using amongst others the pandas internal engine to detect the right value types (int, float, np.Datetime64 etc.)). + dpars = {**dpars, + **pd.read_fwf(io.StringIO(str(string)),widths=[43,1,20],names=['descr','dummy','value']).iloc[1:-1].drop("dummy",1).set_index("descr").T.apply(pd.to_numeric,errors='ignore').iloc[0].to_dict() + } + + # we get weird output when it's a numpy Timestamp, so we convert it to + # pd.datetime type + + dpars['datetime'] = pytz.utc.localize(dt.datetime.strptime(dpars['Observation time'], "%y%m%d/%H%M")) + dpars['STNID'] = dpars['Station number'] + + # altitude above ground level + air_balloon = pd.DataFrame() + air_balloon['z'] = air_balloon_in.HGHT -dpars['Station elevation'] + # absolute humidity in g/kg + air_balloon['q']= (air_balloon_in.MIXR/1000.) \ + / \ + (air_balloon_in.MIXR/1000.+1.) + # convert wind speed from knots to m/s + air_balloon['V'] = 0.51444 * air_balloon_in.SKNT + angle_x = (90.-air_balloon_in.DRCT)/180.*np.pi # assuming that wind in direction of the south is 0 degrees. + + air_balloon['u'] = air_balloon.V * np.sin(angle_x) + air_balloon['v'] = air_balloon.V * np.cos(angle_x) + + + + cp = 1005. # specific heat of dry air [J kg-1 K-1] + Rd = 287. # gas constant for dry air [J kg-1 K-1] + Rv = 461.5 # gas constant for moist air [J kg-1 K-1] + + air_balloon['R'] = (Rd*(1.-air_balloon.q) + Rv*air_balloon.q) + air_balloon['p'] = air_balloon_in.PRES*100. + + air_balloon['t'] = air_balloon_in['TEMP']+273.15 + + # Therefore, determine the sounding that are valid for 'any' column + is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0) + #is_valid = (air_balloon.z >= 0) + # # this is an alternative pipe/numpy method + # (~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)).pipe(np.where)[0] + valid_indices = air_balloon.index[is_valid] + air_balloon = air_balloon[is_valid].reset_index() + #print(valid_indices) + + + if len(air_balloon) > 2: + + dpars['Ps'] = air_balloon.p.values[0] + air_balloon['theta'] = (air_balloon.t) * \ + (dpars['Ps']/(air_balloon.p))**(air_balloon['R']/cp) + air_balloon['thetav'] = air_balloon['theta']*(1. + 0.61 * air_balloon['q']) + + # t_cut_off = 1.5 + # i = 1 + # if t_cut_off is not None: + # + # while ((i < len(air_balloon)) and \ + # ((air_balloon.thetav[0] - air_balloon.thetav[i] ) > t_cut_off)): + # #diff = (air_balloon.theta.iloc[valid_indices[i]] -air_balloon.theta.iloc[valid_indices[i+1]])- 0.5 + # air_balloon.thetav[0:i] = \ + # air_balloon.thetav[i] + t_cut_off + # + # i +=1 + + + + #calculated mixed-layer height considering the critical Richardson number of the virtual temperature profile + dpars['h'],dpars['h_u'],dpars['h_l'] = blh(air_balloon.z,air_balloon.thetav,air_balloon.V) + + dpars['h_b'] = np.max((dpars['h'],10.)) + dpars['h_u'] = np.max((dpars['h_u'],10.)) #upper limit of mixed layer height + dpars['h_l'] = np.max((dpars['h_l'],10.)) #low limit of mixed layer height + dpars['h_e'] = np.abs( dpars['h_u'] - dpars['h_l']) # error of mixed-layer height + + # the final mixed-layer height that will be used by class. We round it + # to 1 decimal so that we get a clean yaml output format + dpars['h'] = np.round(dpars['h_'+air_ap_mode],1) + else: + dpars['h_u'] =np.nan + dpars['h_l'] =np.nan + dpars['h_e'] =np.nan + dpars['h'] =np.nan + dpars['Ps'] = np.nan + air_balloon['t'] = np.nan + air_balloon['theta'] = np.nan + air_balloon['thetav'] = np.nan + + if ~np.isnan(dpars['h']): + # determine mixed-layer properties (moisture, potential temperature...) from profile + + # ... and those of the mixed layer + is_valid_below_h = (air_balloon.z < dpars['h']) + valid_indices_below_h = air_balloon.index[is_valid_below_h].values + if len(valid_indices_below_h) >= 2.: + ml_mean = air_balloon[is_valid_below_h].mean() + else: + ml_mean = air_balloon.iloc[0:1].mean() + + + dpars['theta']= ml_mean.theta + dpars['q'] = ml_mean.q + dpars['u'] = ml_mean.u + dpars['v'] = ml_mean.v + else: + dpars['theta'] = np.nan + dpars['q'] = np.nan + dpars['u'] = np.nan + dpars['v'] = np.nan + + # First 3 data points of the mixed-layer fit. We create a empty head + # first + air_ap_head = air_balloon[0:0] #pd.DataFrame(columns = air_balloon.columns) + # All other data points above the mixed-layer fit + air_ap_tail = air_balloon[air_balloon.z > dpars['h']] + + #calculate mixed-layer jump ( this should be larger than 0.1) + + air_ap_head['z'] = pd.Series(np.array([2.,dpars['h'],dpars['h']])) + # air_ap_head['HGHT'] = air_ap_head['z'] \ + # + \ + # np.round(dpars[ 'Station elevation'],1) + + # make a row object for defining the jump + jump = air_ap_head.iloc[0] * np.nan + + if air_ap_tail.shape[0] > 1: + + # we originally used THTA, but that has another definition than the + # variable theta that we need which should be the temperature that + # one would have if brought to surface (NOT reference) pressure. + for column in ['theta','q','u','v']: + + # initialize the profile head with the mixed-layer values + air_ap_head[column] = ml_mean[column] + # calculate jump values at mixed-layer height, which will be + # added to the third datapoint of the profile head + jump[column] = (air_ap_tail[column].iloc[1]\ + -\ + air_ap_tail[column].iloc[0])\ + /\ + (air_ap_tail.z.iloc[1]\ + - air_ap_tail.z.iloc[0])\ + *\ + (dpars['h']- air_ap_tail.z.iloc[0])\ + +\ + air_ap_tail[column].iloc[0]\ + -\ + ml_mean[column] + if column == 'theta': + # for potential temperature, we need to set a lower limit to + # avoid the model to crash + jump.theta = np.max((0.1,jump.theta)) + + air_ap_head[column][2] += jump[column] + + air_ap_head.V = np.sqrt(air_ap_head.u**2 +air_ap_head.v**2) + + + + # make theta increase strong enough to avoid numerical + # instability + air_ap_tail_orig = pd.DataFrame(air_ap_tail) + air_ap_tail = pd.DataFrame() + #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) + #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) + theta_low = dpars['theta'] + z_low = dpars['h'] + ibottom = 0 + for itop in range(0,len(air_ap_tail_orig)): + theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean() + z_mean = air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean() + if ( + (z_mean > (z_low+10.)) and \ + (theta_mean > (theta_low+0.2) ) and \ + (((theta_mean - theta_low)/(z_mean - z_low)) > 0.0001)): + + air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True) + ibottom = itop+1 + theta_low = air_ap_tail.theta.iloc[-1] + z_low = air_ap_tail.z.iloc[-1] + # elif (itop > len(air_ap_tail_orig)-10): + # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True) + + + + + + air_ap = \ + pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1) + + # we copy the pressure at ground level from balloon sounding. The + # pressure at mixed-layer height will be determined internally by class + #print(air_ap['PRES'].iloc[0]) + + rho = 1.2 # density of air [kg m-3] + g = 9.81 # gravity acceleration [m s-2] + + air_ap['p'].iloc[0] =dpars['Ps'] + air_ap['p'].iloc[1] =(dpars['Ps'] - rho * g * dpars['h']) + air_ap['p'].iloc[2] =(dpars['Ps'] - rho * g * dpars['h'] -0.1) + + + dpars['lat'] = dpars['Station latitude'] + dpars['latitude'] = dpars['lat'] + + # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich) + dpars['lon'] = 0. + # this is the real longitude that will be used to extract ground data + dpars['longitude'] = dpars['Station longitude'] + + dpars['ldatetime'] = dpars['datetime'] \ + + \ + dt.timedelta(minutes=int(dpars['longitude']/360.*24.*60.)) + dpars['doy'] = dpars['datetime'].timetuple().tm_yday + dpars['SolarAltitude'] = \ + GetAltitude(\ + dpars['latitude'],\ + dpars['longitude'],\ + dpars['datetime']\ + ) + dpars['SolarAzimuth'] = GetAzimuth(\ + dpars['latitude'],\ + dpars['longitude'],\ + dpars['datetime']\ + ) + dpars['lSunrise'], dpars['lSunset'] \ + = GetSunriseSunset(dpars['latitude'], + 0., + dpars['ldatetime']) + #print(dpars['lSunrise']) + dpars['lSunrise'] = dpars['lSunrise'] + dpars['lSunset'] = dpars['lSunset'] + # dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise']) + # dpars['lSunset'] = pytz.utc.localize(dpars['lSunset']) + # This is the nearest datetime when the sun is up (for class) + dpars['ldatetime_daylight'] = \ + np.min(\ + (np.max(\ + (dpars['ldatetime'],\ + dpars['lSunrise'])\ + ),\ + dpars['lSunset']\ + )\ + ) + # apply the same time shift for UTC datetime + dpars['datetime_daylight'] = dpars['datetime'] \ + +\ + (dpars['ldatetime_daylight']\ + -\ + dpars['ldatetime']) + + dpars['doy'] = dpars['datetime'].timetuple().tm_yday + + # We set the starting time to the local sun time, since the model + # thinks we are always at the meridian (lon=0). This way the solar + # radiation is calculated correctly. + dpars['tstart'] = dpars['ldatetime_daylight'].hour \ + + \ + dpars['ldatetime_daylight'].minute/60.\ + + \ + dpars['ldatetime_daylight'].second/3600. + + + # convert numpy types to native python data types. This provides + # cleaner data IO with yaml: + for key,value in dpars.items(): + if type(value).__module__ == 'numpy': + dpars[key] = dpars[key].item() + + # # we make a pars object that is similar to the destination object + # pars = model_input() + # for key,value in dpars.items(): + # pars.__dict__[key] = value + + + # we round the columns to a specified decimal, so that we get a clean + # output format for yaml + decimals = {'p':0, 't':2, 'theta':4, 'z':2, 'q':5, 'V':2, 'u':4, 'v':4} +# + for column,decimal in decimals.items(): + air_balloon[column] = air_balloon[column].round(decimal) + air_ap[column] = air_ap[column].round(decimal) + + # in order to avoid warnings: the ABL values should have the same + # rounding as the values profile. + dpars['h'] = round(dpars['h'],decimals['z']) + dpars['theta'] = round(dpars['theta'],decimals['theta']) + dpars['q'] = round(dpars['q'],decimals['q']) + dpars['u'] = round(dpars['u'],decimals['u']) + dpars['v'] = round(dpars['v'],decimals['v']) + + self.update(source='wyoming',\ + # pars=pars, + pars=dpars,\ + air_balloon=air_balloon,\ + air_ap=air_ap) + + + def get_global_input(self, globaldata,only_keys=None,exclude_keys=None): + + """ + Purpose: This sets copies the parameters from the global datasets into the self (or similar object) + according to the position (lat lon) and the class datetime and timespan + globaldata should be a globaldata multifile object + + Input: + - globaldata: this is the library object + - only_keys: only extract specified keys + - exclude_keys: do not inherit specified keys + """ + classdatetime = np.datetime64(self.pars.datetime_daylight) + classdatetime_stop = np.datetime64(self.pars.datetime_daylight \ + + \ + dt.timedelta(seconds=self.pars.runtime)\ + ) + + + # # list of variables that we get from global ground data + # self.ground_keys = ['fW', 'fB', 'fH', 'fTC', 'alpha', 'z0m', 'z0h', + # 'wsat', 'Tsoil', 'cc', 'T2', 'wg', 'w2', 'wfc', + # 'wwilt', 'DSMW', 'tex_coarse_values', 'tex_medium_values', 'tex_fine_values', 'code_values', + # 'texture', 'itex', 'isoil', 'BR', + # 'b', 'cveg', + # 'C1sat', + # 'C2ref', 'p', 'a', + # ] #globaldata.datasets.keys(): + + # # these are the required class4gl 3d atmospheric input which is not provided by the soundings + # self.atm_keys = ['advtheta_x','advtheta_y','advu_x','advu_y','advv_x','advv_y','advq_x','advq_y','w','p'] + + + if type(globaldata) is not data_global: + raise TypeError("Wrong type of input library") + + # by default, we get all dataset keys + keys = list(globaldata.datasets.keys()) + + #print('keys orig', keys) + + # # In case there is surface pressure, we also calculate the half-level + # # and full-level pressure fields + # if ('sp' in keys): + # keys.append('pfull') + # keys.append('phalf') + + # If specified, we only take the keys that are in only_keys + if only_keys is not None: + cycle_keys = list(keys) + for key in cycle_keys: + if key not in only_keys: + keys.remove(key) + + #print('keys 1', keys) + + # If specified, we take out keys that are in exclude keys + if exclude_keys is not None: + for key in keys: + if key in exclude_keys: + keys.remove(key) + + # We add LAI manually, because it is not listed in the datasets and + #they its retreival is hard coded below based on LAIpixel and cveg + if ('LAIpixel' in keys) and ('cveg' in keys): + keys.append('LAI') + + # we set everything to nan first in the pars section (non-profile parameters + # without lev argument), so that we can check afterwards whether the + # data is well-fetched or not. + + for key in keys: + if not ((key in globaldata.datasets) and \ + (globaldata.datasets[key].page is not None) and \ + ('lev' in globaldata.datasets[key].page[key].dims)): + self.update(source='globaldata',pars={key:np.nan}) + # # we do not check profile input for now. We assume it is + # # available + #else: + # self.update(source='globaldata',air_ac=pd.DataFrame({key:list([np.nan])})) + + #print('keys 2', keys) + print(keys) + + for key in keys: + # If we find it, then we obtain the variables + #print('key 0', key) + if ((key in globaldata.datasets) and \ + (globaldata.datasets[key].page is not None)): + + #print('key 1', key) + # check first whether the dataset has a height coordinate (3d space) + if 'lev' in globaldata.datasets[key].page[key].dims: + + # first, we browse to the correct file that has the current time + if 'time' in list(globaldata.datasets[key].page[key].dims): + globaldata.datasets[key].browse_page(time=classdatetime) + + + if (globaldata.datasets[key].page is not None): + # find longitude and latitude coordinates + ilats = (np.abs(globaldata.datasets[key].page.lat - + self.pars.latitude) < 0.5) + ilons = (np.abs(globaldata.datasets[key].page.lon - + self.pars.longitude) < 0.5) + + # if we have a time dimension, then we look up the required timesteps during the class simulation + if 'time' in list(globaldata.datasets[key].page[key].dims): + + DIST = np.abs((globaldata.datasets[key].page['time'].values - classdatetime)) + + idatetime = np.where((DIST) == np.min(DIST))[0][0] + #print('idatetime',idatetime,globaldata.datasets[key].variables['time'].values[idatetime],classdatetime) + if key not in ['t','u','v','q']: + if ((globaldata.datasets[key].page.variables['time'].values[idatetime] < classdatetime) ): + idatetime += 1 + + DIST = np.abs((globaldata.datasets[key].page['time'].values - classdatetime_stop)) + idatetimeend = np.where((DIST) == np.min(DIST))[0][0] + #print('idatetimeend',idatetimeend,globaldata.datasets[key].variables['time'].values[idatetime],classdatetimeend) + if ((globaldata.datasets[key].page.variables['time'].values[idatetimeend] > classdatetime_stop)): + idatetimeend -= 1 + idatetime = np.min((idatetime,idatetimeend)) + #for gleam, we take the previous day values + + # in case of soil temperature or profile temperature, we take the exact + # timing (which is the morning) + if key in ['t','u','v','q']: + idatetimeend = idatetime + + itimes = range(idatetime,idatetimeend+1) + #print(key,'itimes',itimes) + + + # In case we didn't find any correct time, we take the + # closest one. + if len(itimes) == 0: + + + classdatetimemean = \ + np.datetime64(self.pars.datetime_daylight + \ + dt.timedelta(seconds=int(self.pars.runtime/2.) + )) + + dstimes = globaldata.datasets[key].page.time + time = dstimes.sel(time=classdatetimemean,method='nearest') + itimes = (globaldata.datasets[key].page.time == + time) + + else: + # we don't have a time coordinate so it doesn't matter + # what itimes is + itimes = 0 + + #multiplication by 1 is a trick to remove the array()-type in case of zero dimensions (single value). + + # over which dimensions we take a mean: + dims = globaldata.datasets[key].page[key].dims + namesmean = list(dims) + namesmean.remove('lev') + idxmean = [dims.index(namemean) for namemean in namesmean] + + value = \ + globaldata.datasets[key].page[key].isel(time=itimes, + lat=ilats,lon=ilons).mean(axis=tuple(idxmean)).values * 1. + + # Ideally, source should be equal to the datakey of globaldata.library + # or globaldata.datasets (eg., DSMW, IGBP-DIS, ERA-INTERIM etc.) + # but therefore the globaldata class requires a revision to make this work + self.update(source='globaldata',air_ac=pd.DataFrame({key:list(value)})) + + else: + # this procedure is for reading the ground fields (2d space). + # Actually, the code should be simplified to a similar fasion as the 3d procedure above and tested again. + if 'time' in list(globaldata.datasets[key].page[key].dims): + + # first, we browse to the correct file + #print(key) + globaldata.datasets[key].browse_page(time=classdatetime) + + if globaldata.datasets[key].page is not None: + DIST = \ + np.abs((globaldata.datasets[key].page.variables['lat'].values\ + - self.pars.latitude)) + ilat = np.where((DIST) == np.min(DIST))[0][0] + DIST = \ + np.abs((globaldata.datasets[key].page.variables['lon'].values\ + - self.pars.longitude)) + ilon = np.where((DIST) == np.min(DIST))[0][0] + + DIST = \ + np.abs((globaldata.datasets[key].page.variables['lat'].values\ + - (self.pars.latitude + 0.5))) + ilatmax = np.where((DIST) == np.min(DIST))[0][0] + if globaldata.datasets[key].page.variables['lat'].values[ilatmax] < globaldata.datasets[key].page.variables['lat'].values[ilat]: + ilatmax = ilat + + DIST = \ + np.abs((globaldata.datasets[key].page.variables['lon'].values\ + - (self.pars.longitude + 0.5))) + ilonmax = np.where((DIST) == np.min(DIST))[0][0] + if globaldata.datasets[key].page.variables['lon'].values[ilonmax] < globaldata.datasets[key].page.variables['lon'].values[ilon]: + ilonmax = ilon + + DIST = \ + np.abs((globaldata.datasets[key].page.lat.values\ + - (self.pars.latitude - 0.5))) + ilatmin = np.where((DIST) == np.min(DIST))[0][0] + if globaldata.datasets[key].page.variables['lat'].values[ilatmin] > globaldata.datasets[key].page.variables['lat'].values[ilat]: + ilatmin = ilat + DIST = \ + np.abs((globaldata.datasets[key].page.lon.values\ + - (self.pars.longitude - 0.5))) + ilonmin = np.where((DIST) == np.min(DIST))[0][0] + if globaldata.datasets[key].page.variables['lon'].values[ilonmin] > globaldata.datasets[key].page.variables['lon'].values[ilon]: + ilonmin = ilon + + # for the koeppen climate classification and midsummermonth, we just take nearest + print(key) + if key in ['KGC','midsummermonth','AI']: + ilatrange = range(ilat,ilat+1) + ilonrange = range(ilon,ilon+1) + else: + if ilatmin < ilatmax: + ilatrange = range(ilatmin,ilatmax+1) + else: + ilatrange = range(ilatmax,ilatmin+1) + + if ilonmin < ilonmax: + ilonrange = range(ilonmin,ilonmax+1) + else: + ilonrange = range(ilonmax,ilonmin+1) + + if 'time' in list(globaldata.datasets[key].page[key].dims): + DIST = np.abs((globaldata.datasets[key].page['time'].values - classdatetime)) + + idatetime = np.where((DIST) == np.min(DIST))[0][0] + #print('idatetime',idatetime,globaldata.datasets[key].variables['time'].values[idatetime],classdatetime) + + if key not in ['Tsoil','T2','blptb_daymax','t2m_daymax','t2m','blpt','blpt_afternoon','blh','t2m_afternoon','blh_afternoon','blq','blq_afternoon','blptb','blptb_afternoon','blptw','blptw_afternoon','HI','HI_afternoon','rh100','rh100_afternoon']: + if ((globaldata.datasets[key].page.variables['time'].values[idatetime] < classdatetime) ): + idatetime += 1 + + classdatetimeend = np.datetime64(\ + self.pars.datetime_daylight +\ + dt.timedelta(seconds=self.pars.runtime)\ + ) + DIST = np.abs((globaldata.datasets[key].page['time'].values - classdatetimeend)) + idatetimeend = np.where((DIST) == np.min(DIST))[0][0] + #print('idatetimeend',idatetimeend,globaldata.datasets[key].variables['time'].values[idatetime],classdatetimeend) + if ((globaldata.datasets[key].page.variables['time'].values[idatetimeend] > classdatetimeend)): + idatetimeend -= 1 + idatetime = np.min((idatetime,idatetimeend)) + #for gleam, we take the previous day values + if key in ['wg', 'w2']: + idatetime = idatetime - 1 + idatetimeend = idatetimeend - 1 + + # in case of soil temperature or maximum daytime temperature, we take the exact + # timing (which is the morning) + if key in ['Tsoil','T2','t2m_daymax','t2m','blpt','blpt_afternoon','blh','t2m_afternoon','blh_afternoon','blq','blq_afternoon','blptb','blptb_afternoon','blptw','blptw_afternoon','HI','HI_afternoon','rh100','rh100_afternoon']: + idatetimeend = idatetime + + idts = range(idatetime,idatetimeend+1) + + count = 0 + self.__dict__[key] = 0. + value = 0. + for iilat in ilatrange: + for iilon in ilonrange: + for iidts in idts: + value += np.mean(globaldata.datasets[key].page[key].isel(time=iidts,lat=iilat,lon=iilon,drop=True).values) + count += 1 + value = value/count + self.update(source='globaldata',pars={key:value.item()}) + + else: + + count = 0 + value = 0. + for iilat in ilatrange: + for iilon in ilonrange: + value += np.mean(globaldata.datasets[key].page[key].isel(lat=iilat,lon=iilon,drop=True).values) + count += 1 + value = value/count + + self.update(source='globaldata',pars={key:value.item()}) + + if ('LAIpixel' in keys) and ('cveg' in keys): + self.logger.debug('also update LAI based on LAIpixel and cveg') + # I suppose LAI pixel is already determined in the previous + # procedure. Anyway... + key = 'LAIpixel' + + if globaldata.datasets[key].page is not None: + # first, we browse to the correct file that has the current time + if 'time' in list(globaldata.datasets[key].page[key].dims): + globaldata.datasets[key].browse_page(time=classdatetime) + + DIST = \ + np.abs((globaldata.datasets[key].page.lat.values\ + - self.pars.latitude)) + ilat = np.where((DIST) == np.min(DIST))[0][0] + DIST = \ + np.abs((globaldata.datasets[key].page.lon.values\ + - self.pars.longitude)) + ilon = np.where((DIST) == np.min(DIST))[0][0] + + + DIST = \ + np.abs((globaldata.datasets[key].page.lat.values\ + - (self.pars.latitude + 0.5))) + ilatmax = np.where((DIST) == np.min(DIST))[0][0] + if globaldata.datasets[key].page.variables['lat'].values[ilatmax] < globaldata.datasets[key].page.variables['lat'].values[ilat]: + ilatmax = ilat + + DIST = \ + np.abs((globaldata.datasets[key].page.lon.values \ + - (self.pars.longitude + 0.5))) + ilonmax = np.where((DIST) == np.min(DIST))[0][0] + if globaldata.datasets[key].page.variables['lon'].values[ilonmax] < globaldata.datasets[key].page.variables['lon'].values[ilon]: + ilonmax = ilon + + DIST = \ + np.abs((globaldata.datasets[key].page.lat.values\ + - (self.pars.latitude - 0.5))) + ilatmin = np.where((DIST) == np.min(DIST))[0][0] + if globaldata.datasets[key].page.variables['lat'].values[ilatmin] > globaldata.datasets[key].page.variables['lat'].values[ilat]: + ilatmin = ilat + DIST = \ + np.abs((globaldata.datasets[key].page.lon.values\ + - (self.pars.longitude - 0.5))) + ilonmin = np.where((DIST) == np.min(DIST))[0][0] + if globaldata.datasets[key].page.variables['lon'].values[ilonmin] > globaldata.datasets[key].page.variables['lon'].values[ilon]: + ilonmin = ilon + DIST = np.abs((globaldata.datasets[key].page['time'].values - classdatetime)) + idatetime = np.where((DIST) == np.min(DIST))[0][0] + + + if ilatmin < ilatmax: + ilatrange = range(ilatmin,ilatmax+1) + else: + ilatrange = range(ilatmax,ilatmin+1) + + if ilonmin < ilonmax: + ilonrange = range(ilonmin,ilonmax+1) + else: + ilonrange = range(ilonmax,ilonmin+1) + + #tarray_res = np.zeros(shape=globaldata.datasets[key]['time'].shape) + LAIpixel = 0. + count = 0 + for iilat in [ilat]: #ilatrange + for iilon in [ilon]: #ilonrange + LAIpixel += globaldata.datasets[key].page[key].isel(time = idatetime,lat=iilat,lon=iilon,drop=True).values + + + # if np.isnan(tarray[idatetime]): + # print("interpolating GIMMS LAIpixel nan value") + # + # mask = np.isnan(tarray) + # + # #replace each nan value with a interpolated value + # if np.where(mask)[0].shape[0] < 0.25*mask.shape[0]: + # tarray[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), tarray[~mask]) + # + # else: + # print("Warning. Could not interpolate GIMMS LAIpixel nan value") + + # tarray *= np.nan + + count += 1 + #tarray_res += tarray + LAIpixel = LAIpixel/count + + count = 0 + #tarray = globaldata.keys[dataset][key].isel({'lat':[ilat],'lon':[ilon]}).mean(dim=['lat','lon']).values + + self.update(source='globaldata',pars={'LAIpixel':np.float(LAIpixel)}) + #print('LAIpixel:',self.__dict__['LAIpixel']) + #print('cveg:',self.__dict__['cveg']) + + # finally, we rescale the LAI according to the vegetation + # fraction + value = 0. + if ((self.pars.cveg is not None) and (self.pars.cveg > 0.1)): + value =self.pars.LAIpixel/self.pars.cveg + else: + # in case of small vegetation fraction, we take just a standard + # LAI value. It doesn't have a big influence anyway for + # small vegetation + value = 2. + #print('LAI:',self.__dict__['LAI']) + self.update(source='globaldata',pars={'LAI':value}) + + + # in case we have 'sp', we also calculate the 3d pressure fields at + # full level and half level + if ('sp' in keys) and ('sp' in self.pars.__dict__): + pdAB = pd.read_fwf('/user/data/gent/gvo000/gvo00090/EXT/scripts/ECMWF/ecmwf_coeffs_L60_wrf.txt',header=None,names=['A','B'],index_col=0) + + phalf,pfull =calc_air_ac_pres_L60(self.pars.sp,pdAB.A.values,pdAB.B.values) + + + # # # STRANGE, THIS DOESN'T GIVE REALISTIC VALUES, IT NEEDS TO BE + # # # CHECKED AGAIN SINCE THERE IS SIMILAR STRATEGY USED FOR + # # # CALCULATING THE ADVECTION PROFILES + # # hydrostatic thickness of each model layer + delpdgrav = -(phalf[:-1] - phalf[1:])/grav + # # dz = rhodz/(R * T / pfull) + + + # # subsidence multiplied by density. We calculate the subsidence of + # # the in class itself + # wrho = np.zeros_like(phalf) + # wrho[-1] = 0. + + # for ihlev in range(0,wrho.shape[0]-1): + # # subsidence multiplied by density is the integral of + # # divergences multiplied by the layer thicknessies + # wrho[ihlev] = ((self.air_ac['divU_x'][ihlev:] + \ + # self.air_ac['divU_y'][ihlev:]) * \ + # delpdgrav[ihlev:]).sum() + + + + self.update(source='globaldata',\ + air_ac=pd.DataFrame({'p':list(pfull)})) + self.update(source='globaldata',\ + air_ach=pd.DataFrame({'p':list(phalf)})) + self.update(source='globaldata',\ + air_ac=pd.DataFrame({'delpdgrav':list(delpdgrav)})) + # self.update(source='globaldata',\ + # air_ach=pd.DataFrame({'wrho':list(wrho)})) + + + # def get_idx_in_dataset(self, + # globaldata, + # latspan = 0.5): + # lonspan = 0.5): + # """ + # purpose: + # get the xarray indices that are representative between the starting and + # stopping time of the class simulations + + # input: + # self: definition of the class input + # globaldata: book of class4gl global dataset + # key: key variable in the global dataset + # latspan: the span of the lat coordinate + # lonspan: the span of the lon coordinate + + + # output: + # itimes: time coordinates during of the class simulatios + # lats: + # lons: + # """ + + # # first, we browse to the correct file that has the current time + # if 'time' in list(globaldata.datasets[key].page[key].dims): + # globaldata.datasets[key].browse_page(time=classdatetime) + # + # if (globaldata.datasets[key].page is not None): + # # find longitude and latitude coordinates + # ilats = (np.abs(globaldata.datasets[key].page.lat - + # self.pars.latitude) < latspan) + # # In case we didn't find any latitude in the allowed range, we take the closest one. + # if len(ilats) == 0: + # ilats = np.where(\ + # globaldata.datasets[key].page.lat.isin( + # globaldata.datasets[key].page.lat.sel(lat=self.pars.latitude)\ + # ))[0] + # ilons = (np.abs(globaldata.datasets[key].page.lon - + # self.pars.longitude) < lonspan) + # # In case we didn't find any longitude in the allowed range, we take the closest one. + # if len(ilon) == 0: + # ilon = np.where(\ + # globaldata.datasets[key].page.lon.isin( + # globaldata.datasets[key].page.lon.sel(lon=self.pars.longitude)\ + # ))[0] + # + # # if we have a time dimension, then we look up the required timesteps during the class simulation + # if 'time' in list(globaldata.datasets[key].page[key].dims): + + # DIST = np.abs((globaldata.datasets[key].page['time'].values - classdatetime)) + # + # idatetime = np.where((DIST) == np.min(DIST))[0][0] + # #print('idatetime',idatetime,globaldata.datasets[key].variables['time'].values[idatetime],classdatetime) + # if key not in ['t','u','v','q']: + # if ((globaldata.datasets[key].page.variables['time'].values[idatetime] < classdatetime) ): + # idatetime += 1 + # + # DIST = np.abs((globaldata.datasets[key].page['time'].values - classdatetime_stop)) + # idatetimeend = np.where((DIST) == np.min(DIST))[0][0] + # #print('idatetimeend',idatetimeend,globaldata.datasets[key].variables['time'].values[idatetime],classdatetimeend) + # if ((globaldata.datasets[key].page.variables['time'].values[idatetimeend] > classdatetime_stop)): + # idatetimeend -= 1 + # idatetime = np.min((idatetime,idatetimeend)) + # #for gleam, we take the previous day values + + # # in case of soil temperature, we take the exact + # # timing (which is the morning) + # if key in ['t','u','v','q']: + # idatetimeend = idatetime + # + # itimes = range(idatetime,idatetimeend+1) + # #print(key,'itimes',itimes) + + + # # In case we didn't find any correct time, we take the + # # closest one. + # if len(itimes) == 0: + + + # classdatetimemean = \ + # np.datetime64(self.pars.datetime_daylight + \ + # dt.timedelta(seconds=int(self.pars.runtime/2.) + # )) + + # dstimes = globaldata.datasets[key].page.time + # time = dstimes.sel(time=classdatetimemean,method='nearest') + # itimes = (globaldata.datasets[key].page.time == + # time) + # + # else: + # # we don't have a time coordinate so it doesn't matter + # # what itimes is + # itimes = 0 + + # #multiplication by 1 is a trick to remove the array()-type in case of zero dimensions (single value). + # return itimes,ilats,ilons + + + def query_source(self,var): + """ + purpose: + this procedure returns the name of the data source for a certain + variable + + input: + var: this should be in the format "section:variable", eg., + "pars:h", or "air_ac:theta" + + """ + + for source,vars_in_source in self.sources.items(): + if var in vars_in_source: + return source + + def check_source(self,source,check_only_sections=None,ignore_keys=[]): + """ this procedure checks whether data of a specified source is valid. + + INPUT: + source: the data source we want to check + check_only_sections: a string or list with sections to be checked + OUTPUT: + returns True or False + """ + + # we set source ok to false as soon as we find a invalid input + source_ok = True + + # convert to a single-item list in case of a string + check_only_sections_def = (([check_only_sections]) if \ + type(check_only_sections) is str else \ + check_only_sections) + + if source not in self.sources.keys(): + self.logger.info('Source '+source+' does not exist') + source_ok = False + + for sectiondatakey in self.sources[source]: + section,datakey = sectiondatakey.split(':') + if ((check_only_sections_def is None) or \ + (section in check_only_sections_def)): + checkdatakeys = [] + if type(self.__dict__[section]) is pd.DataFrame: + checkdata = self.__dict__[section] + elif type(self.__dict__[section]) is model_input: + checkdata = self.__dict__[section].__dict__ + + if (datakey not in checkdata): + # self.logger.info('Expected key '+datakey+\ + # ' is not in parameter input') + source_ok = False + elif (datakey not in ignore_keys) and \ + ((checkdata[datakey] is None) or \ + (pd.isnull(checkdata[datakey]) is True)): + + # self.logger.info('Key value of "'+datakey+\ + # '" is invalid: ('+ \ + # str(self.__dict__[section].__dict__[datakey])+')') + source_ok = False + self.logger.warning(datakey+' is invalid: '+ str(checkdata[datakey])) + + return source_ok + + def check_source_globaldata(self): + """ this procedure checks whether all global parameter data is + available, according to the keys in the self.sources""" + + source_globaldata_ok = True + + #self.get_values_air_input() + + # and now we can get the surface values + #class_settings = class4gl_input() + #class_settings.set_air_input(input_atm) + if (pd.isnull(self.pars.lat)) or (pd.isnull(self.pars.lon)): + source_globaldata_ok = False + self.logger.warning('lat is invalid: ('+str(self.pars.lat)+')') + self.logger.warning('or lon is invalid: ('+str(self.pars.lon)+')') + else: + # we only check the ground parameter data (pars section). The + # profile data (air_ap section) are supposed to be valid in any + # case. + source_ok = self.check_source(source='globaldata',\ + check_only_sections=['air_ac',\ + 'air_ap',\ + 'pars'], + ignore_keys=[]) + if not source_ok: + source_globaldata_ok = False + self.logger.warning('something was wrong with the profiles') + + # Additional check: we exclude desert-like + if ((self.pars.cveg is None) or pd.isnull(self.pars.cveg)): + source_globaldata_ok = False + self.logger.warning('cveg is invalid: ('+str(self.pars.cveg)+')') + if ((self.pars.LAI is None) or pd.isnull(self.pars.LAI)): + source_globaldata_ok = False + self.logger.warning('LAI is invalid: ('+str(self.pars.LAI)+')') + elif self.pars.cveg < 0.02: + self.logger.warning('cveg is too low: ('+str(self.pars.cveg)+')') + source_globaldata_ok = False + + return source_globaldata_ok + + def mixed_layer_fit(self,air_ap,source,mode): + """ + Purpose: + make a profile fit and write it to the air_ap section of the + class4gl_input object (self). + Input: + air_ap: input profile + + + """ + + + # Raise an error in case the input stream is not the correct object + # if type(wy_strm) is not wyoming: + # raise TypeError('Not a wyoming type input stream') + + # Let's tell the class_input object that it is a Wyoming fit type + self.air_ap_type = source+'_fit' + # ... and which mode of fitting we apply + self.air_ap_mode = mode + + + # Therefore, determine the sounding that are valid for 'any' column + # is_valid = ~np.isnan(air_ap).any(axis=1) & (air_ap.z >= 0) + + if len(~np.isnan(air_ap).any(axis=1) & (air_ap.z >= 0)) == 0.: + self.logger.warning('Warning, not all profile input is valid! Please check input fields!', air_ap) + + + is_valid = (air_ap.z >= 0) + # # this is an alternative pipe/numpy method + # (~np.isnan(air_ap).any(axis=1) & (air_ap.z >= 0)).pipe(np.where)[0] + valid_indices = air_ap.index[is_valid].values + print(valid_indices) + + + hvalues = {} + if len(valid_indices) > 0: + #calculated mixed-layer height considering the critical Richardson number of the virtual temperature profile + hvalues['h_b'] ,hvalues['h_u'],hvalues['h_l'] = blh(air_ap.z,air_ap.thetav,np.sqrt(air_ap.u**2. + air_ap.v**2.)) + + hvalues['h_b'] = np.max((hvalues['h_b'] ,10.)) + hvalues['h_u'] = np.max((hvalues['h_u'] ,10.)) #upper limit of mixed layer height + hvalues['h_l'] = np.max((hvalues['h_l'] ,10.)) #low limit of mixed layer height + hvalues['h_e'] = np.abs( hvalues['h_u'] - hvalues['h_l'] ) # error of mixed-layer height + + # the final mixed-layer height that will be used by class. We round it + # to 1 decimal so that we get a clean yaml output format + hvalues['h'] = np.round(hvalues['h_'+mode],1) + else: + hvalues['h_u'] =np.nan + hvalues['h_l'] =np.nan + hvalues['h_e'] =np.nan + hvalues['h'] =np.nan + + self.update(source='fit_from_'+source,pars=hvalues) + + if np.isnan(self.pars.h ): + self.pars.Ps = np.nan + + mlvalues = {} + if ~np.isnan(self.pars.h ): + # determine mixed-layer properties (moisture, potential temperature...) from profile + + # ... and those of the mixed layer + is_valid_below_h = is_valid & (air_ap.z < self.pars.h) + valid_indices_below_h = air_ap.index[is_valid_below_h].values + if len(valid_indices) > 1: + if len(valid_indices_below_h) >= 3.: + ml_mean = air_ap[is_valid_below_h].mean() + else: + ml_mean = air_ap.iloc[valid_indices[0]:valid_indices[1]].mean() + elif len(valid_indices) == 1: + ml_mean = (air_ap.iloc[0:1]).mean() + else: + temp = pd.DataFrame(air_ap) + temp.iloc[0] = np.nan + ml_mean = temp + + mlvalues['theta'] = ml_mean.theta + mlvalues['q'] = ml_mean.q + mlvalues['u'] = ml_mean.u + mlvalues['v'] = ml_mean.v + else: + mlvalues['theta'] = np.nan + mlvalues['q'] = np.nan + mlvalues['u'] = np.nan + mlvalues['v'] = np.nan + + + + + # First 3 data points of the mixed-layer fit. We create a empty head + # first + air_ap_head = air_ap[0:0] #pd.DataFrame(columns = air_ap.columns) + # All other data points above the mixed-layer fit + air_ap_tail = air_ap[air_ap.z > self.pars.h ] + + #calculate mixed-layer jump ( this should be larger than 0.1) + + air_ap_head['z'] = pd.Series(np.array([2.,self.pars.h ,self.pars.h ])) + #air_ap_head['HGHT'] = air_ap_head['z'] \ + # + \ + # np.round(dpars[ 'Station elevation'],1) + + # make a row object for defining the jump + jump = air_ap_head.iloc[0] * np.nan + + if air_ap_tail.shape[0] > 1: + + # we originally used THTA, but that has another definition than the + # variable theta that we need which should be the temperature that + # one would have if brought to surface (NOT reference) pressure. + for column in ['theta','q','u','v']: + + # initialize the profile head with the mixed-layer values + air_ap_head[column] = ml_mean[column] + # calculate jump values at mixed-layer height, which will be + # added to the third datapoint of the profile head + jump[column] = (air_ap_tail[column].iloc[1]\ + -\ + air_ap_tail[column].iloc[0])\ + /\ + (air_ap_tail.z.iloc[1]\ + - air_ap_tail.z.iloc[0])\ + *\ + (self.pars.h - air_ap_tail.z.iloc[0])\ + +\ + air_ap_tail[column].iloc[0]\ + -\ + ml_mean[column] + if column == 'theta': + # for potential temperature, we need to set a lower limit to + # avoid the model to crash + jump.theta = np.max((0.1,jump.theta)) + + air_ap_head[column][2] += jump[column] + mlvalues['d'+column] = jump[column] + + self.update(source='fit_from_'+source,pars=mlvalues) + + air_ap_head.V = np.sqrt(air_ap_head.u**2 +air_ap_head.v**2) + + # make theta increase strong enough to avoid numerical + # instability + air_ap_tail_orig = pd.DataFrame(air_ap_tail) + air_ap_tail = pd.DataFrame() + #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) + #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) + theta_low = self.pars.theta + z_low = self.pars.h + ibottom = 0 + for itop in range(0,len(air_ap_tail_orig)): + theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean() + z_mean = air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean() + if ( + (z_mean > (z_low+10.)) and \ + (theta_mean > (theta_low+0.2) ) and \ + (((theta_mean - theta_low)/(z_mean - z_low)) > 0.0001)): + + air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True) + ibottom = itop+1 + theta_low = air_ap_tail.theta.iloc[-1] + z_low = air_ap_tail.z.iloc[-1] + # elif (itop > len(air_ap_tail_orig)-10): + # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True) + + + + + + air_ap = \ + pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1) + + # we copy the pressure at ground level from balloon sounding. The + # pressure at mixed-layer height will be determined internally by class + #print(air_ap['PRES'].iloc[0]) + + rho = 1.2 # density of air [kg m-3] + g = 9.81 # gravity acceleration [m s-2] + + air_ap['p'].iloc[0] =self.pars.Ps + air_ap['p'].iloc[1] =(self.pars.Ps - rho * g * self.pars.h ) + air_ap['p'].iloc[2] =(self.pars.Ps - rho * g * self.pars.h -0.1) + + self.update(source='fit_from_'+source,air_ap=air_ap) + + + +class c4gli_iterator(): + """ this iterator allows to loop through an entire yaml file and load class4gl_input sequentially + + for information/documentation on creating such iterator classes, see: https://stackoverflow.com/questions/19151/build-a-basic-python-iterator + """ + def __init__(self,file): + # take file as IO stream + self.file = file + self.yaml_generator = yaml.load_all(file) + self.current_dict = {} + self.current_class4gl_input = class4gl_input() + separator = self.file.readline() # this is just dummy + self.header = file.readline() + if self.header != '# CLASS4GL record; format version: 0.1\n': + raise NotImplementedError("Wrong format version: '"+self.header+"'") + def __iter__(self): + return self + def __next__(self): + self.current_dict = self.yaml_generator.__next__() + self.current_class4gl_input.load_yaml_dict(self.current_dict) + return self.current_class4gl_input + + + +#get_cape and lift_parcel are adapted from the SkewT package + +class gl_dia(object): + def get_lifted_index(self,timestep=-1): + self.LI = get_lifted_index(self.input.Ps,self.out.T2m[timestep],self.out.q[timestep],self.p_pro,self.theta_pro,endp=50000.) + +#from SkewT +#def get_lcl(startp,startt,startdp,nsteps=101): +# from numpy import interp +# #-------------------------------------------------------------------- +# # Lift a parcel dry adiabatically from startp to LCL. +# # Init temp is startt in K, Init dew point is stwrtdp, +# # pressure levels are in Pa +# #-------------------------------------------------------------------- +# +# assert startdp<=startt +# +# if startdp==startt: +# return np.array([startp]),np.array([startt]),np.array([startdp]), +# +# # Pres=linspace(startp,60000.,nsteps) +# Pres=np.logspace(np.log10(startp),np.log10(60000.),nsteps) +# +# # Lift the dry parcel +# T_dry=(startt)*(Pres/startp)**(Rs_da/Cp_da) +# # Mixing ratio isopleth +# starte=VaporPressure(startdp) +# startw=MixRatio(starte,startp) +# e=Pres*startw/(.622+startw) +# T_iso=243.5/(17.67/np.log(e/6.112)-1.) + degCtoK +# +# # Solve for the intersection of these lines (LCL). +# # interp requires the x argument (argument 2) +# # to be ascending in order! +# P_lcl=interp(0.,T_iso-T_dry,Pres) +# T_lcl=interp(P_lcl,Pres[::-1],T_dry[::-1]) +# +# # # presdry=linspace(startp,P_lcl) +# # presdry=logspace(log10(startp),log10(P_lcl),nsteps) +# +# # tempdry=interp(presdry,Pres[::-1],T_dry[::-1]) +# # tempiso=interp(presdry,Pres[::-1],T_iso[::-1]) +# +# return P_lcl,T_lcl + + + + + + +#from class +def get_lcl(startp,startt,startqv): + # Find lifting condensation level iteratively + lcl = 20. + RHlcl = 0.5 + + itmax = 30 + it = 0 + while(((RHlcl <= 0.9999) or (RHlcl >= 1.0001)) and it 0: + iTHTV_0 = iTHTV_0[0] + THTV_0 = THTV[iTHTV_0] + else: + THTV_0 = np.nan + + RiB = 9.81/THTV_0 * ( THTV - THTV_0) * HAGL / np.clip(WSPD,a_min=0.1,a_max=None)**2. + + + + #RiB = 9.81/THTV_0 * ( THTV[i-1] + (HGHT[i] - HGHT[i-1])/ - THTV_0) * HAGL / WSPD**2 + #RiB - RiBc = 0 + + #best guess of BLH + + #print("RiB: ",RiB) + #print("RiBc: ",RiBc) + + + + BLHi = np.where(RiB > RiBc)[0] + if len(BLHi ) > 0: + BLHi = BLHi[0] + #print("BLHi: ",BLHi) + BLH = (HAGL[BLHi] - HAGL[BLHi-1])/(RiB[BLHi] -RiB[BLHi-1]) * (RiBc - RiB[BLHi-1]) + HAGL[BLHi-1] + + # possible error is calculated as the difference height levels used for the interpolation + BLHu = np.max([BLH,HAGL[BLHi]-eps]) + BLHl = np.min([BLH,HAGL[BLHi-1]+eps]) + # calculate an alternative BLH based on another critical Richardson number (RiBce): + BLHi =np.where(RiB > RiBce)[0] + if len(BLHi ) > 0: + BLHi = BLHi[0] + + BLHa = (HAGL[BLHi] - HAGL[BLHi-1])/(RiB[BLHi] -RiB[BLHi-1]) * (RiBc - RiB[BLHi-1]) + HAGL[BLHi-1] + BLHu = np.max([BLHu,HAGL[BLHi]-eps]) + BLHl = np.min([BLHl,HAGL[BLHi-1]+eps]) + + BLHu = np.max([BLHu,BLH + abs(BLH-BLHa)]) + BLHl = np.min([BLHl,BLH - abs(BLH-BLHa)]) + + else: + BLH,BLHu,BLHl = np.nan, np.nan,np.nan + + else: + BLH,BLHu,BLHl = np.nan, np.nan,np.nan + + return BLH,BLHu,BLHl + +class class4gl(model): + """ the extension of the 'class model' class """ + + def dump(self,file,include_input=False,timeseries_only=None): + """ this procedure dumps the class4gl object into a yaml file + + Input: + - self.__dict__ (internal): the dictionary from which we read + - timeseries_only: for the timeseries output, dump only + specific output variables + Output: + - file: All the parameters in self.__init__() are written to + the yaml file, including pars, air_ap, sources etc. + """ + + if include_input: + self.input_c4gl.dump(file) + + file.write('---\n') + index = file.tell() + file.write('# CLASS4GL input; format version: 0.1\n') + + # write out the position of the current record + yaml.dump({'index':index}, file, default_flow_style=False) + + + # we only copy those variables that are also in the input + # for pars, these variables are in the model object + dictpars = {} + for key in self.__dict__.keys(): + # we omit the profile data and the timeseries in the out section, + # since they are in pandas format for convenience + # We include it hereafter as separate sections in the yaml file + if key not in ['air_ach',\ + 'air_ac',\ + 'air_ap',\ + 'out',\ + 'input',\ + 'logger',\ + 'input_c4gl']: + dictpars[key] = self.__dict__[key] + + # convert numpy types to native python data types. This + # provides cleaner data IO with yaml: + if type(dictpars[key]).__module__ == 'numpy': + dictpars[key] = dictpars[key].item() + + + + + dictout = {} + dictoutlast = {} + if not 'out' in self.__dict__.keys(): + print('Warning: no timeseries section found in output.') + else: + outvars = [] + if timeseries_only == None: + outvars = self.__dict__['out'].__dict__.keys() + else: + outvars = timeseries_only + + for key in outvars: + dictout[key] = self.__dict__['out'].__dict__[key] + dictoutlast[key] = dictout[key][-1] + + if type(dictoutlast[key]).__module__ == 'numpy': + dictoutlast[key] = dictoutlast[key].item() + # convert numpy types to native python data types. This + # provides cleaner data IO with yaml: + if type(dictout[key]).__module__ == 'numpy': + dictout[key] = [ a.item() for a in \ + self.__dict__['out'].__dict__[key]] + #dictout[key] = list(dictout[key] ) + + yaml.dump({'pars' : {**dictoutlast,**dictpars}},file) + + if ('sw_ac' in self.input.__dict__.keys()) \ + and (self.input.__dict__['sw_ac']): + yaml.dump({'air_ac' : \ + self.__dict__['air_ac'].to_dict(orient='list')},file) + #yaml.dump({'air_ach' : \ + # self.__dict__['air_ach'].to_dict(orient='list')},file) + if ('sw_ap' in self.input.__dict__.keys()) and (self.input.__dict__['sw_ap']): + #print('hello',self.air_ap.to_dict(orient='list')) + yaml.dump({'air_ap' : \ + self.__dict__['air_ap'].to_dict(orient='list')},file) + + yaml.dump({'out' : dictout},file) + diff --git a/class4gl/data_global.py b/class4gl/data_global.py new file mode 100644 index 0000000..564f0f1 --- /dev/null +++ b/class4gl/data_global.py @@ -0,0 +1,950 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Tue Nov 7 10:51:03 2017 + +@author: Hendrik Wouters + +Purpose: provides class routines for ground and atmosphere conditions used for +the CLASS miced-layer model + +Usage: + from data_global import data_global + from class4gl import class4gl_input + from data_soundings import wyoming + + # create a data_global object and load initial data pages + globaldata = data_global() + globaldata.load_datasets() + # create a class4gl_input object + c4gli = class4gl_input() + # Initialize it with profile data. We need to do this first. Actually this + # will set the coordinate parameters (datetime, latitude, longitude) in + # class4gl_input.pars.__dict__, which is required to read point data from + # the data_global object. + + # open a Wyoming stream for a specific station + wy_strm = wyoming(STNM=91376) + # load the first profile + wy_strm.find_first() + # load the profile data into the class4gl_input object + c4gli.get_profile_wyoming(wy_strm) + + # and finally, read the global input data for this profile + c4gli.get_global_input(globaldata) + + +""" + +import netCDF4 as nc4 +import numpy as np +import datetime as dt +#you can install with +#import pynacolada as pcd +import pandas as pd +import xarray as xr +import os +import glob +import sys +import errno +import warnings +import logging + + +#formatter = logging.Formatter() +logging.basicConfig(format='%(asctime)s - \ + %(name)s - \ + %(levelname)s - \ + %(message)s') + +class book(object): + """ this is a class for a dataset spread over multiple files. It has a + similar purpose open_mfdataset, but only 1 file (called current 'page') + one is loaded at a time. This saves precious memory. """ + def __init__(self,fn,concat_dim = None,debug_level=None): + self.logger = logging.getLogger('book') + if debug_level is not None: + self.logger.setLevel(debug_level) + + # filenames are expanded as a list and sorted by filename + self.pages = glob.glob(fn); self.pages.sort() + # In case length of the resulting list is zero, this means no file was found that matches fn. In that case we raise an error. + if len(self.pages) == 0: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), fn) + self.ipage = -1; self.page = None + self.renames = {} # each time when opening a file, a renaming should be done. + self.set_page(0) + + # we consider that the outer dimension is the one we concatenate + self.concat_dim = concat_dim + if self.concat_dim is None: + self.concat_dim = self.concat_dim=list(self.page.dims.keys())[0] + + # this wraps the xarray sel-commmand + def sel(*args, **kwargs): + for dim in kwargs.keys(): + if dim == self.concat_dim: + self.browse_page(**{dim: kwargs[dim]}) + return page.sel(*args,**kwargs) + + + ## this wraps the xarray class -> some issues with that, so I just copy the sel command (which I do not use yet) + #def __getattr__(self,attr): + # orig_attr = self.page.__getattribute__(attr) + # if callable(orig_attr): + # def hooked(*args, **kwargs): + # for dim in kwargs.keys(): + # if dim == self.concat_dim: + # self.browse_page(**{dim: kwargs[dim]}) + # + # result = orig_attr(*args, **kwargs) + # # prevent wrapped_class from becoming unwrapped + # if result == self.page: + # return self + # self.post() + # return result + # return hooked + # else: + # return orig_attr + + def set_renames(self,renames): + #first, we convert back to original names, and afterwards, we apply the update of the renames. + reverse_renames = dict((v,k) for k,v in self.renames.items()) + self.renames = renames + if self.page is not None: + self.page = self.page.rename(reverse_renames) + self.page = self.page.rename(self.renames) + + def set_page(self,ipage,page=None): + """ this sets the right page according to ipage: + - We do not switch the page if we are already at the right one + - we set the correct renamings (level -> lev, latitude -> lat, + etc.) + - The dataset is also squeezed. + """ + + if ((ipage != self.ipage) or (page is not None)): + + if self.page is not None: + self.page.close() + + self.ipage = ipage + if page is not None: + self.page = page + else: + if self.ipage == -1: + self.page = None + else: + #try: + + self.logger.info("Switching to page "+str(self.ipage)+': '\ + +self.pages[self.ipage]) + self.page = xr.open_dataset(self.pages[self.ipage]) + + + # do some final corrections to the dataset to make them uniform + if self.page is not None: + if 'latitude' in self.page.dims: +# sel f.library[fn] = self.library[fn].rename({'latitude':'lat','longitude':'lon'}) + + self.page = self.page.rename({'latitude':'lat','longitude':'lon'}) + if 'level' in self.page.dims: + self.page = self.page.rename({'level':'lev'}) + + lon = self.page.lon.values + lon[lon > 180.] -= 360. + self.page.lon.values[:] = lon[:] + + + self.page = self.page.rename(self.renames) + self.page = self.page.squeeze(drop=True) + + def browse_page(self,rewind=2,**args): + + # at the moment, this is only tested with files that are stacked according to the time dimension. + dims = args.keys() + + + if self.ipage == -1: + self.set_page(0) + + found = False + iipage = 0 + startipage = self.ipage - rewind + while (iipage < len(self.pages)) and not found: + ipage = (iipage+startipage) % len(self.pages) + for dim in args.keys(): + this_file = True + + # here we store the datetimes in a directly-readable dictionary, so that we don't need to load it every time again + if 'dims' not in self.__dict__: + self.dims = {} + if dim not in self.dims.keys(): + self.dims[dim] = [None]*len(self.pages) + + if self.dims[dim][ipage] is None: + self.logger.info('Loading coordinates of dimension "'+dim+\ + '" of page "' +str(ipage)+'".') + self.set_page(ipage) + # print(ipage) + # print(dim) + # print(dim,self.page[dim].values) + self.dims[dim][ipage] = self.page[dim].values + + # determine current time range of the current page + mindim = self.dims[dim][ipage][0] -(self.dims[dim][ipage][1] - self.dims[dim][ipage][0])/2. + maxdim = self.dims[dim][ipage][-1] +(self.dims[dim][ipage][-1] - self.dims[dim][ipage][-2])/2. + + if not ((args[dim] >= mindim) and (args[dim] < maxdim )): + this_file = False + + if this_file: + found = True + self.set_page(ipage) + else: + + #if ((args[dim] >= self.page[dim].min().values) and (args[dim] < self.page[dim].max().values)): + # iipage = len(self.pages) # we stop searching + + iipage += 1 + + if not found: + self.logger.info("Page not found. Setting to page -1") + #iipage = len(self.pages) # we stop searching further + self.set_page(-1) + + if self.ipage != -1: + self.logger.debug("I'm now at page "+ str(self.ipage)+': '+self.pages[self.ipage]) + else: + self.logger.debug("I'm now at page "+ str(self.ipage)) + +class data_global(object): + def __init__(self,sources= { + 'KOEPPEN:KGC' : '/user/data/gent/gvo000/gvo00090/EXT/data/KOEPPEN/Koeppen-Geiger.nc', + # # old gleam + # 'GLEAM:wg' : '/user/data/gent/gvo000/gvo00090/GLEAM/data/v3.1a/????/SMsurf_*_GLEAM_v3.1a.nc:SMsurf', + # 'GLEAM:w2' : '/user/data/gent/gvo000/gvo00090/GLEAM/data/v3.1a/????/SMroot_*_GLEAM_v3.1a.nc:SMroot', + # 'GLEAM:BR' : '/user/data/gent/gvo000/gvo00090/GLEAM/data/v3.1a/????/BR_*_GLEAM_v3.1a.nc:BR', + # 'GLEAM:EF' : '/user/data/gent/gvo000/gvo00090/GLEAM/data/v3.1a/????/EF_*_GLEAM_v3.1a.nc:EF', + # 'GLEAM:wg' : '/user/data/gent/gvo000/gvo00090/GLEAM/data/GLEAM_v3.2/v3.2a_OUTPUT/????/SMsurf_*_GLEAM_v3.2a.nc:SMsurf', + # 'GLEAM:w2' : '/user/data/gent/gvo000/gvo00090/GLEAM/data/GLEAM_v3.2/v3.2a_OUTPUT/????/SMroot_*_GLEAM_v3.2a.nc:SMroot', + #'GLEAM:BR' : '/user/data/gent/gvo000/gvo00090/GLEAM/data/GLEAM_v3.2/v3.2a/????/BR_*_GLEAM_v3.2a.nc:BR', + # 'GLEAM:EF' : '/user/data/gent/gvo000/gvo00090/GLEAM/data/GLEAM_v3.2/v3.2a_OUTPUT/????/EF_*_GLEAM_v3.2a.nc:EF', + "IGBPDIS:alpha" : "/user/data/gent/gvo000/gvo00090/EXT/data/IGBP-DIS/FRACTIONS_GLEAMv31a.nc", + "GLAS:z0m" : "/user/data/gent/gvo000/gvo00090/EXT/data/GLAS/global_canopy_height_0.25.nc:Band1", + "GLAS:z0h" : "/user/data/gent/gvo000/gvo00090/EXT/data/GLAS/global_canopy_height_0.25.nc:Band1", + 'IGBPDIS:wsat' : '/user/data/gent/gvo000/gvo00090/EXT/data/IGBP-DIS/wsat.nc', + "ERAINT:Ts" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/stl1_3hourly_xarray/stl1*_3hourly.nc:stl1", + "ERAINT:Tsoil" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/stl1_3hourly_xarray/stl1*_3hourly.nc:stl1", + "ERAINT:T2" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/stl2_3hourly_xarray/stl2*_3hourly.nc:stl2", + "ERAINT:cc" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/tcc_3hourly_xarray/tcc*_3hourly.nc:tcc", + 'IGBPDIS:wfc' : '/user/data/gent/gvo000/gvo00090/EXT/data/IGBP-DIS/wfc.nc', + 'IGBPDIS:wwilt' : '/user/data/gent/gvo000/gvo00090/EXT/data/IGBP-DIS/wwp.nc:wwp', + 'MOD44B:cveg' : '/user/data/gent/gvo000/gvo00090/EXT/data/MOD44B/fv.nc:fv', + #'CERES:cc' : '/user/data/gent/gvo000/gvo00090/EXT/data/CERES/CERES_SYN1deg-1H_Terra-Aqua-MODIS_Ed4A_Subset*.nc:cldarea_total_1h', + "DSMW:b" : "/user/data/gent/gvo000/gvo00090/EXT/data/DSMW/FAO_DSMW_DP.nc:DSMW:b", + #"DSMW.C1sat" : "/user/data/gent/gvo000/gvo00090/EXT/data/DSMW/FAO_DSMW_DP.nc:DSMW:C1sat", + #"DSMW.C2ref" : "/user/data/gent/gvo000/gvo00090/EXT/data/DSMW/FAO_DSMW_DP.nc:DSMW:C2ref", + #"DSMW.p" : "/user/data/gent/gvo000/gvo00090/EXT/data/DSMW/FAO_DSMW_DP.nc:DSMW:p", + #"DSMW.a" : "/user/data/gent/gvo000/gvo00090/EXT/data/DSMW/FAO_DSMW_DP.nc:DSMW:a", + #"DSMW.CGsat" : "/user/data/gent/gvo000/gvo00090/EXT/data/DSMW/FAO_DSMW_DP.nc:DSMW:CGsat", + "GIMMS:LAIpixel": "/user/data/gent/gvo000/gvo00090/EXT/data/GIMMS/v2/LAI/gimms-3g.v2.lai.1981-2015_monmean_remapcon_0.25.nc:LAI", + #'CERES.low': '/user/data/gent/gvo000/gvo00090/vsc42247/EXT/data/CERES/CERES_SYN1deg-1H_Terra-Aqua-MODIS_Ed4A_Subset_*.nc%cldarea_low_1h', + #'CERES.cc%20000301%20100101': '/user/data/gent/gvo000/gvo00090/vsc42247/EXT/data/CERES/CERES_SYN1deg-1H_Terra-Aqua-MODIS_Ed4A_Subset_$YYYYMMDD_CERES_START-$YYYYMMDD_CERES_END.nc.cldarea_total_1h%cldarea_total_1h' +# "ERAINT:advt_x" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/advt_x_6hourly/advt_x*_6hourly.nc:advt_x", +# "ERAINT:advt_y" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/advt_y_6hourly/advt_y*_6hourly.nc:advt_y", +# "ERAINT:advq_x" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/advq_x_6hourly/advq_x*_6hourly.nc", +# "ERAINT:advq_y" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/advq_y_6hourly/advq_y*_6hourly.nc", +# "ERAINT:advu_x" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/advu_x_6hourly/advu_x*_6hourly.nc", +# "ERAINT:advu_y" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/advu_y_6hourly/advu_y*_6hourly.nc", +# "ERAINT:advv_x" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/advv_x_6hourly/advv_x*_6hourly.nc", +# "ERAINT:advv_y" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/advv_y_6hourly/advv_y*_6hourly.nc", + #"ERAINT:divU_x" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/divU_x_6hourly/divU_x*_6hourly.nc:__xarray_dataarray_variable__", + #"ERAINT:divU_y" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/divU_y_6hourly/divU_y*_6hourly.nc:__xarray_dataarray_variable__", + "ERAINT:sp" : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/sp_6hourly/sp_*_6hourly.nc", + "ERAINT:wp" : '/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/w_6hourly_xarray/w*_6hourly.nc:w', + "MSWEPGLEAM:AI" : '/data/gent/vo/000/gvo00090/D2D/data/Aridity//Ep_1981_2017_MO_meanhottestmonth.nc', +# "ERA5:t2m_daymax" : '/data/gent/vo/000/gvo00090/D2D/data/ERA5/by_var_nc/t2m_1hourly_for_t2m_daymax.nc:t2m', + "ERA5:blpt_daymax" : '/scratch/gent/vo/000/gvo00090/D2D/data/ERA5/by_var_nc/blpt_1hourly_for_blpt_daymax.nc:blpt', + "ERA5:blptb_daymax" : '/scratch/gent/vo/000/gvo00090/D2D/data/ERA5/by_var_nc/blptb_1hourly_for_blptb_daymax.nc:blptb', + "ERA5:blptb_daymean" : '/scratch/gent/vo/000/gvo00090/D2D/data/ERA5/by_var_nc/blptb_1hourly_daymean.nc:blptb', + "ERA5:blptb_daymean_ge90" : '/scratch/gent/vo/000/gvo00090/D2D/data/ERA5/by_var_nc/blptb_1hourly_daymean_for_blptb_daymean_mean3month_is_max_ge90.nc:blptb', + + # "ERA5:slhf" : '/user/data/gent/gvo000/gvo00090/EXT/data/ERA5/by_var_nc/slhf_1hourly/slhf_*_1hourly.nc', + # "ERA5:sshf" : '/user/data/gent/gvo000/gvo00090/EXT/data/ERA5/by_var_nc/sshf_1hourly/sshf_*_1hourly.nc', + #"MSWEP:pr" :"/user/data/gent/gvo000/gvo00090/EXT/data/MSWEP/MSWEP_v1.2_precip_1979-2015/3hr/raw_data/globe/*.nc:precipitation" + },debug_level=None): + self.library = {} #unique references to data sources being used. They can be files that are original on the disks or some unambiguous xarray virtual sources. These references are used in other variables. This way, a file or source cannot be loaded twice (a warning is made if one would try it). + self.sources = sources + self.datarefs = {} + self.datasets = {} + self.datetime = dt.datetime(1981,1,1) + + self.logger = logging.getLogger('data_global') + if debug_level is not None: + self.logger.setLevel(debug_level) + self.debug_level = debug_level + + warnings.warn('omitting pressure field p and advection') + + def in_library(self,fn): + if fn not in self.library.keys(): + return False + else: + print("Warning: "+fn+" is already in the library.") + return True + + def add_to_library(self,fn): + if not self.in_library(fn): + print("opening: "+fn) + self.library[fn] = \ + book(fn,concat_dim='time',debug_level=self.debug_level) + + #self.library[fn] = xr.open_mfdataset(fn,concat_dim='time') + #if 'latitude' in self.library[fn].variables: + # self.library[fn] = self.library[fn].rename({'latitude':'lat','longitude':'lon'}) + + + # default procedure for loading datasets into the globaldata library + def load_dataset_default(self,input_fn,varssource=None,varsdest=None): + if type(varssource) is str: + varssource = [varssource] + if type(varsdest) is str: + varsdest = [varsdest] + + self.add_to_library(input_fn) + + if varssource is None: + varssource = [] + for var in self.sources[input_fn].variables: + avoid = \ + ['lat','lon','latitude','longitude','time','lev','level'] + if ((len(list(var.shape)) >= 2) & (var not in avoid)): #two-dimensional array + varssource.append(var) + + if varsdest is None: + varsdest = varssource + + #input_fn = "/user/data/gent/gvo000/gvo00090/EXT/data/IGBP-DIS/wsat.nc" + for ivar,vardest in enumerate(varsdest): + varsource = varssource[ivar] + print('setting '+vardest+' as '+varsource+' from '+input_fn) + + if vardest in self.datarefs.keys(): + print("Warning! "+vardest+' is already provided by ',self.datarefs[vardest]+'. \n Overwriting....') + #self.add_to_library(fn,varsource,vardest) + if vardest != varsource: + libkey = input_fn+'.'+varsource+'.'+vardest + if libkey not in self.library.keys(): + #self.library[libkey] = self.library[input_fn].rename({varsource:vardest}) + self.library[libkey] = book(input_fn,\ + debug_level=self.debug_level) + self.library[libkey].set_renames({varsource: vardest}) + + self.datarefs[vardest] = libkey # this is to remember that it was originally varsource in input_fn + self.datasets[vardest] =self.library[self.datarefs[vardest]] + else: + self.datarefs[vardest] = input_fn + self.datasets[vardest] =self.library[self.datarefs[vardest]] + + # if ((vardest is not None) & (vardest not in self.datasets[vardest].variables)): + # print('Warning: '+ vardest "not in " + input_fn) + + + + def load_datasets(self,sources = None,recalc=0): + + if sources is None: + sources = self.sources + for key in sources.keys(): + #datakey,vardest,*args = key.split(':') + datakey,vardest = key.split(':') + #print(datakey) + + fnvarsource = sources[key].split(':') + if len(fnvarsource) > 2: + #fn,varsource,*fnargs = fnvarsource + fn,varsource,fnargs = fnvarsource + fnargs = [fnargs] + elif len(fnvarsource) > 1: + #fn,varsource,*fnargs = fnvarsource + fn,varsource = fnvarsource + fnargs = [] + else: + fn = sources[key] + varsource = vardest + self.load_dataset(fn,varsource,vardest,datakey,recalc=recalc) + + def load_dataset(self,fn,varsource,vardest,datakey,recalc=0): + # the default way of loading a 2d dataset + if datakey in ['CERES','GLEAM','ERAINT','GIMMS']: + self.load_dataset_default(fn,varsource,vardest) + elif datakey == 'IGBPDIS': + if vardest == 'alpha': + ltypes = ['W','B','H','TC'] + for ltype in ltypes: + self.load_dataset_default(fn,'f'+ltype,'f'+ltype) + ##self.datasets['f'+ltype]['f'+ltype]= self.datasets['f'+ltype]['f'+ltype].squeeze(drop=True) + + + # landfr = {} + # for ltype in ['W','B','H','TC']: + # landfr[ltype] = datasets['f'+ltype]['f'+ltype].values + + + + keytemp = 'alpha' + fnkeytemp = fn+':IGBPDIS:alpha' + if (os.path.isfile(fnkeytemp)) and ( recalc < 6): + self.library[fnkeytemp] = book(fnkeytemp, + debug_level=self.debug_level) + self.datasets[keytemp] = self.library[fnkeytemp] + self.datarefs[keytemp] = fnkeytemp + else: + self.library[fn+':IGBPDIS:alpha'] = xr.Dataset() + #self.library[fn+':IGBPDIS:alpha'][keytemp] = xr.zeros_like(self.datasets['IGBPDIS']['IGBPDIS'],dtype=np.float)*np.nan + self.library[fn+':IGBPDIS:alpha']['lat'] = self.datasets['fW'].page['lat'] + self.library[fn+':IGBPDIS:alpha']['lon'] = self.datasets['fW'].page['lon'] + self.library[fn+':IGBPDIS:alpha'][keytemp] = xr.DataArray(np.zeros(shape=(self.datasets['fW'].page['lon'].shape[0],self.datasets['fW'].page['lat'].shape[0]),dtype=np.float),dims=('lon','lat')) + self.datasets[keytemp] = self.library[fn+':IGBPDIS:alpha'] + self.datarefs[keytemp] =fn+':IGBPDIS:alpha' + + aweights = {'W':0.075,'TC':0.15,'H':0.22,'B':0.30} + + alpha=self.library[fn+':IGBPDIS:alpha'][keytemp].values + for ltype in ltypes: + alpha += self.datasets['f'+ltype].page['f'+ltype].values*aweights[ltype] + + self.library[fn+':IGBPDIS:alpha'][keytemp].values = alpha + print('writing file to: '+fnkeytemp) + os.system('rm '+fnkeytemp) + self.library[fnkeytemp].to_netcdf(fnkeytemp) + self.library[fnkeytemp].close() + + + self.library[fnkeytemp] = \ + book(fnkeytemp,debug_level=self.debug_level) + self.datasets[keytemp] = self.library[fnkeytemp] + self.datarefs[keytemp] = fnkeytemp + + + else: + self.load_dataset_default(fn,varsource,vardest) + + + elif datakey == 'GLAS': + self.load_dataset_default(fn,varsource,vardest) + if vardest == 'z0m': + self.datasets['z0m'].page['z0m'].values = (self.datasets['z0m'].page['z0m'].values/10.).clip(0.01,None) + elif vardest == 'z0h': + self.datasets['z0h'].page['z0h'].values = (self.datasets['z0h'].page['z0h'].values/100.).clip(0.001,None) + elif datakey == 'DSMW': + + + # Procedure of the thermal properties: + # 1. determine soil texture from DSMW/10. + # 2. soil type with look-up table (according to DWD/EXTPAR) + # 3. Thermal properties used in the force-restore method (Clapp and Hornberger, 1987) + # with parameter look-up table from Noilhan and Planton (1989). + # Note: The look-up table is inspired on DWD/COSMO + + # to do: implement inheretance, so that the the preliminary output of DSMW or any other dataset can be calculated first + + + + fnout = fn.replace('*','') # for storing computationally heavy soil properties, instead of calculating everytime + self.load_dataset_default(fn,'DSMW') + print('calculating texture') + SPKEYS = ['tex_coarse', 'tex_medium', 'tex_fine', 'code','undefined'] + TEMP = {} + TEMP2 = self.datasets['DSMW'].page['DSMW'].values + TEMP3 = {} + for SPKEY in SPKEYS: + + + keytemp = SPKEY+'_values' + fnoutkeytemp = fnout+':DSMW:'+keytemp + if (os.path.isfile(fnoutkeytemp)) and ( recalc < 5 ): + self.library[fn+':DSMW:'+SPKEY+'_values'] = \ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets[SPKEY+'_values'] = self.library[fn+':DSMW:'+SPKEY+'_values'] + self.datarefs[SPKEY+'_values'] =fn+':DSMW:'+SPKEY+'_values' + + + else: + #DSMW = self.datasets['DSMW']['DSMW']# self.input_nc.variables['DSMW'][ilat,ilon] + self.library[fn+':DSMW:'+SPKEY+'_values'] = xr.Dataset() + self.library[fn+':DSMW:'+SPKEY+'_values']['lat'] = self.datasets['DSMW'].page['lat'] + self.library[fn+':DSMW:'+SPKEY+'_values']['lon'] = self.datasets['DSMW'].page['lon'] + self.library[fn+':DSMW:'+SPKEY+'_values'][SPKEY+'_values'] = xr.DataArray(np.zeros(shape=(self.datasets['DSMW'].page['lat'].shape[0],self.datasets['DSMW'].page['lon'].shape[0]),dtype=np.int),dims=('lat','lon')) + #self.library[fn+':DSMW:'+SPKEY+'_values'][SPKEY+'_values'] = xr.zeros_like(self.datasets['DSMW']['DSMW'],dtype=(np.int if SPKEY == 'code' else np.float)) + self.datasets[SPKEY+'_values'] = self.library[fn+':DSMW:'+SPKEY+'_values'] + self.datarefs[SPKEY+'_values'] =fn+':DSMW:'+SPKEY+'_values' + + # for faster computation, we need to get it to memory out of Dask. + TEMP[SPKEY] = self.datasets[SPKEY+'_values'][SPKEY+'_values'].values + TEMP3[SPKEY] = self.datasets['DSMW'].page[SPKEY].values + + # yes, I know I only check the last file. + if not ((os.path.isfile(fnoutkeytemp)) and ( recalc < 5)): + for idx in range(len(self.datasets['DSMW'].page['tex_coarse'].values))[:]: + print('idx',idx,SPKEY) + SEL = (TEMP2 == idx) + # print(idx,len(TEMP3)) + for SPKEY in SPKEYS: + TEMP[SPKEY][SEL] = TEMP3[SPKEY][idx] + + for SPKEY in SPKEYS: + keytemp = SPKEY+'_values' + fnoutkeytemp = fnout+':DSMW:'+keytemp + self.datasets[SPKEY+'_values'][SPKEY+'_values'].values = TEMP[SPKEY][:] + os.system('rm '+fnoutkeytemp) + self.datasets[SPKEY+'_values'].to_netcdf(fnoutkeytemp) + self.datasets[SPKEY+'_values'].close() + + + self.library[fn+':DSMW:'+SPKEY+'_values'] = \ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets[SPKEY+'_values'] = self.library[fn+':DSMW:'+SPKEY+'_values'] + self.datarefs[SPKEY+'_values'] =fn+':DSMW:'+SPKEY+'_values' + + + keytemp = 'texture' + fnoutkeytemp=fnout+':DSMW:'+keytemp + if (os.path.isfile(fnoutkeytemp)) and ( recalc < 3 ): + self.library[fnoutkeytemp] = \ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets[keytemp] = self.library[fn+':DSMW:texture'] + self.datarefs[keytemp] =fn+':DSMW:texture' + else: + self.library[fn+':DSMW:texture'] = xr.Dataset() + #self.library[fn+':DSMW:texture'][keytemp] = xr.zeros_like(self.datasets['DSMW']['DSMW'],dtype=np.float)*np.nan + self.library[fn+':DSMW:texture']['lat'] = self.datasets['DSMW'].page['lat'] + self.library[fn+':DSMW:texture']['lon'] = self.datasets['DSMW'].page['lon'] + self.library[fn+':DSMW:texture'][keytemp] = xr.DataArray(np.zeros(shape=(self.datasets['DSMW'].page['lat'].shape[0],self.datasets['DSMW'].page['lon'].shape[0]),dtype=np.float),dims=('lat','lon')) + self.datasets[keytemp] = self.library[fn+':DSMW:texture'] + self.datarefs[keytemp] =fn+':DSMW:texture' + + + + self.datasets[keytemp][keytemp].values = (0.5*self.datasets['tex_medium_values'].page['tex_medium_values'].values+1.0*self.datasets['tex_coarse_values'].page['tex_coarse_values'].values)/(self.datasets['tex_coarse_values'].page['tex_coarse_values'].values+self.datasets['tex_medium_values'].page['tex_medium_values'].values+self.datasets['tex_fine_values'].page['tex_fine_values'].values) + + zundef = np.array(self.datasets['undefined_values'].page['undefined_values'].values,dtype=np.float) + zundef[zundef < 0] = np.nan + zsum_tex = self.datasets['tex_coarse_values'].page['tex_coarse_values'].values+self.datasets['tex_medium_values'].page['tex_medium_values'].values+ self.datasets['tex_fine_values'].page['tex_fine_values'].values + VALID = (zsum_tex >= zundef) *( ~np.isnan(zundef)) + + self.datasets[keytemp][keytemp].values[~VALID] = 9012. + + os.system('rm '+fnoutkeytemp) + self.datasets[keytemp].to_netcdf(fnoutkeytemp) + self.datasets[keytemp].close() + + + self.library[fnoutkeytemp] = \ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets[keytemp] = self.library[fn+':DSMW:texture'] + self.datarefs[keytemp] =fn+':DSMW:texture' + + + print('calculating texture type') + + + + keytemp = 'itex' + fnoutkeytemp=fnout+':DSMW:'+keytemp + if (os.path.isfile(fnoutkeytemp)) and ( recalc < 2 ): + self.library[fnoutkeytemp] = \ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets[keytemp] = self.library[fn+':DSMW:itex'] + self.datarefs[keytemp] =fn+':DSMW:itex' + else: + self.library[fnoutkeytemp] = xr.Dataset() + self.library[fnoutkeytemp][keytemp] = xr.zeros_like(self.datasets['DSMW'].page['DSMW'],dtype=np.int) + self.datasets[keytemp] = self.library[fn+':DSMW:itex'] + self.datarefs[keytemp] =fn+':DSMW:itex' + + X = self.datasets['texture'].page['texture'].values*100 + X[pd.isnull(X)] = -9 + + + self.datasets[keytemp][keytemp].values = X + + os.system('rm '+fnoutkeytemp) + self.datasets['itex'].to_netcdf(fnoutkeytemp) + self.datasets['itex'].close() + + + self.library[fnoutkeytemp] = \ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets[keytemp] = self.library[fn+':DSMW:itex'] + self.datarefs[keytemp] =fn+':DSMW:itex' + + + keytemp = 'isoil' + fnoutkeytemp=fnout+':DSMW:'+keytemp + isoil_reprocessed = False + if (os.path.isfile(fnoutkeytemp)) and ( recalc < 1): + self.library[fn+':DSMW:isoil'] = \ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets['isoil'] = self.library[fn+':DSMW:isoil'] + self.datarefs['isoil'] =fn+':DSMW:isoil' + else: + isoil_reprocessed = True + print('calculating soil type') + self.library[fn+':DSMW:isoil'] = xr.Dataset() + self.library[fn+':DSMW:isoil']['isoil'] = xr.zeros_like(self.datasets['DSMW'].page['DSMW'],dtype=np.int) + self.datasets['isoil'] = self.library[fn+':DSMW:isoil'] + self.datarefs['isoil'] =fn+':DSMW:isoil' + + #adopted from mo_agg_soil.f90 (EXTPAR3.0) + self.datasets['isoil']['isoil'] = xr.zeros_like(self.datasets['DSMW'].page['DSMW'],dtype=np.int) + ITEX = self.datasets['itex'].page['itex'].values + ISOIL = 9 + 0.*self.datasets['isoil']['isoil'].values + LOOKUP = [ + [-10 ,9],# ocean + [0 ,7],# fine textured, clay (soil type 7) + [20,6],# medium to fine textured, loamy clay (soil type 6) + [40,5],# medium textured, loam (soil type 5) + [60,4],# coarse to medium textured, sandy loam (soil type 4) + [80,3],# coarse textured, sand (soil type 3) + [100,9],# coarse textured, sand (soil type 3) + ] + for iitex,iisoil in LOOKUP: + ISOIL[ITEX > iitex] = iisoil + print('iitex,iisoil',iitex,iisoil) + + + #adopted from mo_agg_soil.f90 (EXTPAR3.0) + LOOKUP = [ + [9001, 1 ], # ice, glacier (soil type 1) + [9002, 2 ], # rock, lithosols (soil type 2) + [9003, 3 ], # salt, set soiltype to sand (soil type 3) + [9004, 8 ], # histosol, e.g. peat (soil type 8) + [9, 9 ], # undefined (ocean) + [9005, 3 ], # shifting sands or dunes, set soiltype to sand (soil type 3) + [9000, 9 ], # undefined (inland lake) + [9009, 5 ], # default_soiltype ! undefined (nodata), set soiltype to loam (soil type ) + [9012, 5 ], # default_soiltype undefined (dominant part undefined), set soiltype to loam (soil type 5) + ] + # EXTPAR: soil_code = soil_texslo(soil_unit)%dsmw_code # the legend has some special cases for the "soil_code" + CODE_VALUES = self.datasets['code_values'].page['code_values'].values + + CODE_VALUES[ITEX == 901200] = 9012 + for icode,iisoil in LOOKUP: + ISOIL[CODE_VALUES == icode] = iisoil + + self.datasets['isoil']['isoil'].values = ISOIL + os.system('rm '+fnoutkeytemp) + self.datasets[keytemp].to_netcdf(fnoutkeytemp) + self.datasets[keytemp].close() + print('saved inbetween file to: '+fnoutkeytemp) + + self.library[fn+':DSMW:isoil'] = \ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets['isoil'] = self.library[fn+':DSMW:isoil'] + self.datarefs['isoil'] =fn+':DSMW:isoil' + + #adopted from data_soil.f90 (COSMO5.0) + SP_LOOKUP = { + # soil type: ice rock sand sandy loam clay clay peat sea sea + # (by index) loam loam water ice + 'cporv' : [ np.nan, 1.E-10 , 1.E-10 , 0.364 , 0.445 , 0.455 , 0.475 , 0.507 , 0.863 , 1.E-10 , 1.E-10 ], + 'cfcap' : [ np.nan, 1.E-10 , 1.E-10 , 0.196 , 0.260 , 0.340 , 0.370 , 0.463 , 0.763 , 1.E-10 , 1.E-10 ], + 'cpwp' : [ np.nan, 0.0 , 0.0 , 0.042 , 0.100 , 0.110 , 0.185 , 0.257 , 0.265 , 0.0 , 0.0 ], + 'cadp' : [ np.nan, 0.0 , 0.0 , 0.012 , 0.030 , 0.035 , 0.060 , 0.065 , 0.098 , 0.0 , 0.0 ], + 'crhoc' : [ np.nan, 1.92E6 , 2.10E6 , 1.28E6 , 1.35E6 , 1.42E6 , 1.50E6 , 1.63E6 , 0.58E6 , 4.18E6 , 1.92E6 ], + 'cik2' : [ np.nan, 0.0 , 0.0 , 0.0035 , 0.0023 , 0.0010 , 0.0006 , 0.0001 , 0.0002 , 0.0 , 0.0 ], + 'ckw0' : [ np.nan, 0.0 , 0.0 , 479.E-7 , 943.E-8 , 531.E-8 , 764.E-9 , 17.E-9 , 58.E-9 , 0.0 , 0.0 ], + 'ckw1' : [ np.nan, 0.0 , 0.0 , -19.27 , -20.86 , -19.66 , -18.52 , -16.32 , -16.48 , 0.0 , 0.0 ], + 'cdw0' : [ np.nan, 0.0 , 0.0 , 184.E-7 , 346.E-8 , 357.E-8 , 118.E-8 , 442.E-9 , 106.E-9 , 0.0 , 0.0 ], + 'cdw1' : [ np.nan, 0.0 , 0.0 , -8.45 , -9.47 , -7.44 , -7.76 , -6.74 , -5.97 , 0.0 , 0.0 ], + 'crock' : [ np.nan, 0.0 , 0.0 , 1.0 , 1.0 , 1.0 , 1.0 , 1.0 , 1.0 , 0.0 , 0.0 ], + 'cala0' : [ np.nan, 2.26 , 2.41 , 0.30 , 0.28 , 0.25 , 0.21 , 0.18 , 0.06 , 1.0 , 2.26 ], + 'cala1' : [ np.nan, 2.26 , 2.41 , 2.40 , 2.40 , 1.58 , 1.55 , 1.50 , 0.50 , 1.0 , 2.26 ], + 'csalb' : [ np.nan, 0.70 , 0.30 , 0.30 , 0.25 , 0.25 , 0.25 , 0.25 , 0.20 , 0.07 , 0.70 ], + 'csalbw' : [ np.nan, 0.00 , 0.00 , 0.44 , 0.27 , 0.24 , 0.23 , 0.22 , 0.10 , 0.00 , 0.00 ], + 'ck0di' : [ np.nan, 1.E-4 , 1.E-4 , 2.E-4 , 2.E-5 , 6.E-6 , 2.E-6 , 1.E-6 , 1.5E-6 , 0.00 , 0.00 ], + 'cbedi' : [ np.nan, 1.00 , 1.00 , 3.5 , 4.8 , 6.1 , 8.6 , 10.0 , 9.0 , 0.00 , 0.00 ], + 'csandf' : [ np.nan, 0.0 , 0.0 , 90. , 65. , 40. , 35. , 15. , 90. , 0.00 , 0.00 ], + 'cclayf' : [ np.nan, 0.0 , 0.0 , 5.0 , 10. , 20. , 35. , 70. , 5.0 , 0.00 , 0.00 ], + # Important note: For peat, the unknown values below are set equal to that of loam + #supplement Noihhan andf Planton 1989 soil texture parameters for the force-restore method. + 'b' : [ np.nan, np.nan , np.nan , 4.05 , 4.90 , 5.39 , 8.52 , 11.40 , 5.39 , np.nan , np.nan ], + #error in table 2 of NP89: values need to be multiplied by e-6 + 'CGsat' : [ np.nan, np.nan , np.nan , 3.222e-6 , 3.560e-6 , 4.111e-6 , 3.995e-6 , 3.600e-6 , np.nan , np.nan , np.nan ], + 'p' : [ np.nan, np.nan , np.nan , 4. , 4. , 6. , 10. , 12. , 6. , np.nan , np.nan ], + + 'a' : [ np.nan, np.nan , np.nan , 0.387 , 0.219 , 0.148 , 0.084 , 0.083 , 0.148 , np.nan , np.nan ], + 'C1sat' : [ np.nan, np.nan , np.nan , 0.082 , 0.132 , 0.191 , 0.227 , 0.342 , 0.191 , np.nan , np.nan ], + 'C2ref' : [ np.nan, np.nan , np.nan , 3.9 , 1.8 , 0.8 , 0.6 , 0.3 , 0.8 , np.nan , np.nan ], + } + + + # isoil_reprocessed = False + # if (os.path.isfile(fnoutkeytemp)) and ( recalc < 1): + + # self.library[fn+':DSMW:isoil'] = \ + # book(fnoutkeytemp,debug_level=self.debug_level) + # self.datasets['isoil'] = self.library[fn+':DSMW:isoil'] + # self.datarefs['isoil'] =fn+':DSMW:isoil' + # else: + # isoil_reprocessed = True + # print('calculating soil type') + # self.library[fn+':DSMW:isoil'] = xr.Dataset() + # self.library[fn+':DSMW:isoil']['isoil'] = xr.zeros_like(self.datasets['DSMW'].page['DSMW'],dtype=np.int) + # self.datasets['isoil'] = self.library[fn+':DSMW:isoil'] + # self.datarefs['isoil'] =fn+':DSMW:isoil' + + + + + # this should become cleaner in future but let's hard code it for now. + DSMWVARS = ["b", "C1sat","C2ref","p","a" ] + print('calculating soil parameter') + DATATEMPSPKEY = {} + if (recalc < 1) and (isoil_reprocessed == False): + for SPKEY in DSMWVARS:#SP_LOOKUP.keys(): + keytemp = SPKEY + fnoutkeytemp=fnout+':DSMW:'+keytemp + self.library[fn+':DSMW:'+SPKEY] =\ + book(fnoutkeytemp,debug_level=self.debug_level) + self.datasets[SPKEY] = self.library[fnoutkeytemp] + self.datarefs[SPKEY] =fnoutkeytemp + else: + for SPKEY in DSMWVARS:#SP_LOOKUP.keys(): + + self.library[fn+':DSMW:'+SPKEY] = xr.Dataset() + self.library[fn+':DSMW:'+SPKEY][SPKEY] = xr.zeros_like(self.datasets['DSMW'].page['DSMW'],dtype=np.float) + self.datasets[SPKEY] = self.library[fn+':DSMW:'+SPKEY] + self.datarefs[SPKEY] =fn+':DSMW:'+SPKEY + DATATEMPSPKEY[SPKEY] = self.datasets[SPKEY][SPKEY].values + ISOIL = self.datasets['isoil'].page['isoil'].values + print(np.where(ISOIL>0.)) + for i in range(11): + SELECT = (ISOIL == i) + for SPKEY in DSMWVARS:#SP_LOOKUP.keys(): + DATATEMPSPKEY[SPKEY][SELECT] = SP_LOOKUP[SPKEY][i] + + for SPKEY in DSMWVARS:#SP_LOOKUP.keys(): + self.datasets[SPKEY][SPKEY].values = DATATEMPSPKEY[SPKEY] + + os.system('rm '+fn+':DSMW:'+SPKEY) + self.datasets[SPKEY].to_netcdf(fn+':DSMW:'+SPKEY) + self.datasets[SPKEY].close() + print('saved inbetween file to: '+fn+':DSMW:'+SPKEY) + + self.library[fn+':DSMW:'+SPKEY] = \ + book(fn+':DSMW:'+SPKEY,debug_level=self.debug_level) + self.datasets[SPKEY] = self.library[fn+':DSMW:'+SPKEY] + self.datarefs[SPKEY] =fn+':DSMW:'+SPKEY + + + else: + self.load_dataset_default(fn,varsource,vardest) + + + + + + +# +# # only print the last parameter value in the plot +# +# #inputs.append(cp.deepcopy(class_settings)) +# #var = 'cala' +# #class_settings.__dict__[var] = np.float(SP['cala0']) +# #valnew = class_settings.__dict__[var] +# #labels.append(var+': '+format(valold,"0.2g")+'->'+format(valnew,"0.2g")) +# +# #inputs.append(cp.deepcopy(class_settings)) +# #var = 'crhoc' +# #class_settings.__dict__[var] = np.float(SP['crhoc']) +# #valnew = class_settings.__dict__[var] +# #labels.append(var+': '+format(valold,"0.2g")+'->'+format(valnew,"0.2g")) +# +# key = "CERES" +# if ((kwargs == {}) or ((key in kwargs.keys()) and (kwargs[key]))): +# +# CERES_start_date = dt.datetime(2000,3,1) +# DT_CERES_START = (CERES_start_date + dt.timedelta(days=(int((class_settings.datetime - CERES_start_date ).days/61) * 61))) +# DT_CERES_END = DT_CERES_START +dt.timedelta(days=60) +# +# input_fn = "/user/data/gent/gvo000/gvo00090/EXT/data/CERES/CERES_SYN1deg-1H_Terra-Aqua-MODIS_Ed4A_Subset_"+DT_CERES_START.strftime("%Y%m%d")+"-"+DT_CERES_END.strftime("%Y%m%d")+".nc" +# print("Reading afternoon cloud cover for "+str(class_settings.datetime)+" from "+input_fn) +# +# var = 'cc' +# +# input_nc = nc4.Dataset(input_fn,'r') +# +# idatetime = np.where(np.array(pcd.ncgetdatetime(input_nc)) >= class_settings.datetime)[0][0] +# idatetime_end = np.where(np.array(pcd.ncgetdatetime(input_nc)) < (class_settings.datetime+dt.timedelta(hours=int(class_settings.runtime/3600.))))[0][-1] +# +# ilat = np.where(input_nc.variables['lat'][:] >= class_settings.lat)[0][-1] +# ilon = np.where(input_nc.variables['lon'][:] >= class_settings.lon)[0][0] +# print(class_settings.lat,class_settings.lon) +# +# class_settings.__dict__[var] = np.nanmean(input_nc.variables['cldarea_total_1h'][idatetime:idatetime_end,ilat,ilon])/100. +# +# input_nc.close() +# + + +# key = "GIMMS" +# if ((kwargs == {}) or ((key in kwargs.keys()) and (kwargs[key]))): +# +# +# input_fn = "/user/data/gent/gvo000/gvo00090/EXT/data/GIMMS/v2/LAI/gimms-3g.v2.lai.1981-2015_monmean.nc" +# print("Reading Leag Area Index from "+input_fn) +# var = 'LAI' +# +# #plt.plot +# +# input_nc = nc4.Dataset(input_fn,'r') +# +# #idatetime = np.where(np.array(pcd.ncgetdatetime(input_nc)) >= class_settings.datetime)[0][0] +# idatetime = np.where(np.array(pcd.ncgetdatetime(input_nc)) >= class_settings.datetime)[0][0] +# +# ilatitude = np.where(input_nc.variables['lat'][:] >= class_settings.lat)[0][-1] +# ilongitude = np.where(input_nc.variables['lon'][:] >= class_settings.lon)[0][0] +# +# # divide by cveg, since it only reflects the LAI for the vegetation fraction and not for the entire (satellite) grid cell +# +# print('Warning! Dividing by cveg, which is: '+str(class_settings.cveg)) +# tarray = np.array(input_nc.variables['LAI'][:,ilatitude,ilongitude])/class_settings.cveg +# +# if np.isnan(tarray[idatetime]): +# print("interpolating GIMMS cveg nan value") +# +# mask = np.isnan(tarray) +# if np.where(mask)[0].shape[0] < 0.25*mask.shape[0]: +# tarray[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), tarray[~mask]) +# else: +# print("Warning. Could not interpolate GIMMS cveg nan value") +# +# class_settings.__dict__[var] = tarray[idatetime] +# +# input_nc.close() +# +# key = "IGBPDIS_ALPHA" +# if ((kwargs == {}) or ((key in kwargs.keys()) and (kwargs[key]))): +# +# var = 'alpha' +# +# input_fn = "/user/data/gent/gvo000/gvo00090/EXT/data/IGBP-DIS/FRACTIONS_GLEAMv31a.nc" +# print("Reading albedo from "+input_fn) +# +# input_nc = nc4.Dataset(input_fn,'r') +# ilat = np.where(input_nc.variables['lat'][:] >= class_settings.lat)[0][-1] +# ilon = np.where(input_nc.variables['lon'][:] >= class_settings.lon)[0][0] +# +# +# landfr = {} +# for ltype in ['W','B','H','TC']: +# landfr[ltype] = input_nc.variables['f'+ltype][0,ilon,ilat] +# +# aweights = {'W':0.075,'TC':0.15,'H':0.22,'B':0.30} +# +# alpha=0. +# for ltype in landfr.keys(): +# alpha += landfr[ltype]*aweights[ltype] +# +# +# class_settings.__dict__[var] = alpha +# input_nc.close() +# +# +# key = "ERAINT_ST" +# if ((kwargs == {}) or ((key in kwargs.keys()) and (kwargs[key]))): +# +# input_fn = '/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/stl1_3hourly/stl1_'+str(class_settings.datetime.year)+"_3hourly.nc" +# print("Reading soil temperature from "+input_fn) +# +# var = 'Tsoil' +# input_nc = nc4.Dataset(input_fn,'r') +# +# idatetime = np.where(np.array(pcd.ncgetdatetime(input_nc)) >= class_settings.datetime)[0][0] +# +# ilatitude = np.where(input_nc.variables['latitude'][:] >= class_settings.lat)[0][-1] +# ilongitude = np.where(input_nc.variables['longitude'][:] >= class_settings.lon)[0][0] +# +# +# class_settings.__dict__[var] = input_nc.variables['stl1'][idatetime,ilatitude,ilongitude] +# +# input_fn = '/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/stl2_3hourly/stl2_'+str(class_settings.datetime.year)+"_3hourly.nc" +# var = 'T2' +# +# input_nc = nc4.Dataset(input_fn,'r') +# +# idatetime = np.where(np.array(pcd.ncgetdatetime(input_nc)) >= class_settings.datetime)[0][0] +# +# ilatitude = np.where(input_nc.variables['latitude'][:] >= class_settings.lat)[0][-1] +# ilongitude = np.where(input_nc.variables['longitude'][:] >= class_settings.lon)[0][0] +# +# +# class_settings.__dict__[var] = input_nc.variables['stl2'][idatetime,ilatitude,ilongitude] +# +# +# input_nc.close() +# +# +# +# #inputs.append(cp.deepcopy(class_settings)) +# #var = 'T2' +# #valold = class_settings.__dict__[var] +# # +# #class_settings.__dict__[var] = 305. +# #class_settings.__dict__['Tsoil'] = 302. +# #valnew = class_settings.__dict__[var] +# #labels.append(var+': '+format(valold,"0.2g")+'->'+format(valnew,"0.2g")) +# +# +# +# #inputs.append(cp.deepcopy(class_settings)) +# # +# #var = 'Lambda' +# #valold = class_settings.__dict__[var] +# +# ## I presume that the skin layer conductivity scales with both LAI and vegetation fraction, which seems ~ valid according to table 10.6 in CLASS-book. +# ## I need to ask Chiel. +# ## I extrapolate from Lambda value of grass with Lambda = 5.9 W m-2 K-1, LAI = 2 and cveg = 0.85 +# # +# #valnew = 5.9 / 2. / 0.85 * class_settings.__dict__['LAI'] * class_settings.__dict__['cveg'] +# #class_settings.__dict__[var] = valnew +# #labels.append(var+': '+format(valold,"0.2g")+'->'+format(valnew,"0.2g")) +# +# +# +# key = "GLAS" +# if ((kwargs == {}) or ((key in kwargs.keys()) and (kwargs[key]))): +# +# input_fn = "/user/data/gent/gvo000/gvo00090/EXT/data/GLAS/global_canopy_height_0.25.nc" +# print("Reading canopy height for determining roughness length from "+input_fn) +# var = 'z0m' +# +# +# #plt.plot +# +# input_nc = nc4.Dataset(input_fn,'r') +# +# ilat = np.where(input_nc.variables['lat'][:] >= class_settings.lat)[0][0] +# ilon = np.where(input_nc.variables['lon'][:] >= class_settings.lon)[0][0] +# +# testval = np.float64(input_nc.variables['Band1'][ilat,ilon])/10. +# +# lowerlimit = 0.01 +# if testval < lowerlimit: +# print('forest canopy height very very small. We take a value of '+str(lowerlimit)) +# class_settings.__dict__[var] = lowerlimit +# else: +# class_settings.__dict__[var] = testval +# +# class_settings.__dict__['z0h'] = class_settings.__dict__['z0m']/10. +# +# +# input_nc.close() + + + + + diff --git a/class4gl/data_soundings.py b/class4gl/data_soundings.py new file mode 100644 index 0000000..8e438b4 --- /dev/null +++ b/class4gl/data_soundings.py @@ -0,0 +1,453 @@ +import numpy as np + +from bs4 import BeautifulSoup +import pandas as pd +import datetime as dt +#import pylab as pl +import io +import os +import calendar +import warnings + + + +def dtrange(STARTTIME,ENDTIME,TIMEJUMP=dt.timedelta(hours=24)): + STEPS = int((ENDTIME - STARTTIME).total_seconds()/TIMEJUMP.total_seconds()) + return [STARTTIME + TIMEJUMP*i for i in range(0,STEPS)] + + +#from os import listdir +#from os.path import isfile #,join +import glob + + +class wyoming(object): + def __init__(self, PATH="/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/",STNM=None): + + self.profile_type = 'wyoming' + self.MONTHS = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'] + self.PATH = PATH + self.reset() + if STNM is not None: + self.set_STNM(STNM) + else: + warnings.warn('warning. No station is set yet. Please use class function set_STNM to set station number') + + def reset(self): + + self.status = 'init' + self.found = False + self.DT = None + self.current = None + #self.mode = 'b' + + + def set_STNM(self,STNM): + self.reset() + self.STNM = STNM + self.FILES = glob.glob(self.PATH+'/????/SOUNDINGS_????_'+format(STNM,'05d')+".html") + self.FILES = [os.path.realpath(FILE) for FILE in self.FILES] + self.current = None + self.found = False + self.FILES.sort() + + def find_first(self,year=None,get_atm=False): + self.found = False + self.current = None + + # check first file/year or specified year + if year == None: + self.iFN = 0 + self.FN = self.FILES[self.iFN] + else: + # this counter cycles through all the years. We avoid an endless + # loop. We don't check more years than the amount of self.FILES + iyear = 0 + while ((not self.found) and (iyear < 100)): + + FN = os.path.realpath(self.PATH \ + +'/' \ + +str(year+iyear) \ + +'/SOUNDINGS_' \ + +str(year+iyear) \ + +'_' \ + +format(self.STNM,'05d') \ + +".html") + if FN in self.FILES: + self.iFN = self.FILES.index(FN) + self.found = True + self.FN = FN + else: + self.iFN = -1 + self.FN = None + + iyear += 1 + + if self.found: + self.sounding_series = BeautifulSoup(open(self.FN), "html.parser") + self.current = self.sounding_series.find('h2') + keepsearching = (self.current is None) #if we don't want later years, add here: "and (year is None)" + + # go through other files and find first sounding when year is not specified + self.iFN=self.iFN+1 + while keepsearching: + self.FN = self.FILES[self.iFN] + self.sounding_series = BeautifulSoup(open(self.FN), "html.parser") + self.current = self.sounding_series.find('h2') + self.iFN=self.iFN+1 + keepsearching = (self.current is None) and (self.iFN < len(self.FILES)) + self.found = (self.current is not None) + + self.status = 'fetch' + if self.found: + self.DT = dt.datetime(int(self.current.text[-4:]),self.MONTHS.index(self.current.text[-8:-5])+1,int(self.current.text[-11:-9]),int(self.current.text[-15:-13])) + + # if self.found and get_atm: + # self.get_values_air_input() + + + def find(self,DT,get_atm=False): + + self.found = False + keepsearching = True + #print(DT) + # we open a new file only when it's needed. Otherwise we just scroll to the right sounding. + if not ((self.current is not None) and (DT >= self.DT) and (self.DT.year == DT.year)): + self.DT = DT + self.FN = os.path.realpath(self.PATH+"/"+self.DT.strftime("%Y")+"/SOUNDINGS_"+self.DT.strftime("%Y")+"_"+format(self.STNM,'05d')+".html") + self.iFN = self.FILES.index(self.FN) + self.sounding_series = BeautifulSoup(open(self.FN), "html.parser") + self.current = self.sounding_series.find('h2') + + keepsearching = (self.current is not None) + while keepsearching: + DTcurrent = dt.datetime(int(self.current.text[-4:]),self.MONTHS.index(self.current.text[-8:-5])+1,int(self.current.text[-11:-9]),int(self.current.text[-15:-13])) + if DTcurrent == DT: + self.found = True + keepsearching = False + # if get_atm: + # self.get_values_air_input() + self.DT = DTcurrent #dt.datetime(int(self.current.text[-4:]),self.MONTHS.index(self.current.text[-8:-5])+1,int(self.current.text[-11:-9]),int(self.current.text[-15:-13])) + elif DTcurrent > DT: + keepsearching = False + self.current = None + else: + self.current = self.current.find_next('h2') + if self.current is None: + keepsearching = False + self.found = (self.current is not None) + self.status = 'fetch' + + def find_next(self,get_atm=False): + self.found = False + self.DT = None + if self.current is None: + self.find_first() + else: + self.current = self.current.find_next('h2') + self.found = (self.current is not None) + keepsearching = ((self.current is None) and ((self.iFN+1) < len(self.FILES))) + while keepsearching: + self.iFN=self.iFN+1 + self.FN = self.FILES[self.iFN] + self.sounding_series = BeautifulSoup(open(self.FN), "html.parser") + self.current = self.sounding_series.find('h2') + + self.found = (self.current is not None) + keepsearching = ((self.current is None) and ((self.iFN+1) < len(self.FILES))) + if self.found: + self.DT = dt.datetime(int(self.current.text[-4:]),self.MONTHS.index(self.current.text[-8:-5])+1,int(self.current.text[-11:-9]),int(self.current.text[-15:-13])) + # if self.found and get_atm: + # self.get_values_air_input() + + + + +# # should be placed under class4gl!!! +# class sounding(object): +# #def statistics: +# # returns a list of sounding statistics as a dict + + +#def get_sounding_wyoming(self,wy_strm,latitude=None,longitude=None): +# input: +# wy_strm: wyoming stream + +# for iDT,DT in enumerate(DTS): + + #websource = urllib.request.urlopen(webpage) +#soup = BeautifulSoup(open(webpage), "html.parser") + +# __init__(self): + +# sounding_keys +# list of variables that we get from global ground data + +#workaround for ...last line has 
 which results in stringlike first column
+
+
+
+
+
+# dtheta_pre = air
+# 
+# dtheta_pre = (new_pro_h[1] - new_pro_h[0])/(listHAGLNEW[4] - listHAGLNEW[3])*(BLH-listHAGLNEW[3]) + new_pro_h[0] - meanabl 
+# 
+# 
+# dtheta = np.max((0.1,dtheta_pre))
+# 
+# 
+# 
+# 
+# self.air_ap.
+
+
+
+
+# if len(valid_indices) > 0:
+#     #print('valid_indices',valid_indices)
+#     if len(np.where(HAGL[valid_indices[0]:] < BLH)[0]) >= 3:
+#         meanabl = np.nanmean(np.array(ONE_COLUMN[col][HAGL < BLH][(valid_indices[0]+1):],dtype=np.float))
+#     else:
+#         meanabl = np.nanmean(ONE_COLUMN[col][valid_indices[0]:(valid_indices[0]+1)],dtype=np.float)                    
+# else:
+#     meanabl = np.nanmean(ONE_COLUMN[col][0:1],dtype=np.float)
+# #
+
+
+
+
+
+
+
+# # fit new profiles taking the above-estimated mixed-layer height
+# ONE_COLUMNNEW = []
+# for BLH in [self.h,self.h_u,self.h_d]:
+#     ONE_COLUMNNEW.append(pd.DataFrame())
+#     
+#     HAGLNEW = np.array([2.,BLH,BLH]+list(HAGL[HAGL > BLH]),dtype=np.float)
+#     ONE_COLUMNNEW[-1].insert(0,'HAGL',HAGLNEW)
+#     
+#     listHAGLNEW = list(HAGLNEW)
+#     for icol,col in enumerate(['THTA','THTV','QABS','SKNT','DRCT','PRES']):
+#         
+#         # get index of lowest valid observation. This seems to vary
+#         idxvalid = np.where((np.array(HAGL) >= 0) & (~pd.isnull(np.array(ONE_COLUMN[col],dtype=np.float) )))[0]
+#         if len(idxvalid) > 0:
+#             #print('idxvalid',idxvalid)
+#             if len(np.where(HAGL[idxvalid[0]:] < BLH)[0]) >= 3:
+#                 meanabl = np.nanmean(np.array(ONE_COLUMN[col][HAGL < BLH][(idxvalid[0]+1):],dtype=np.float))
+#             else:
+#                 meanabl = np.nanmean(ONE_COLUMN[col][idxvalid[0]:(idxvalid[0]+1)],dtype=np.float)                    
+#         else:
+#             meanabl = np.nanmean(ONE_COLUMN[col][0:1],dtype=np.float)
+#             #print(col,meanabl)
+#        
+#         
+#         # if col == 'PRES':
+#         #     meanabl =  
+#     
+#         new_pro_h = list(np.array(ONE_COLUMN[col][HAGL > BLH],dtype=np.float))
+#         #THTVM = np.nanmean(THTV[HAGL <= BLH])
+#         #print("new_pro_h",new_pro_h)
+#         # calculate jump ath the top of the mixed layer
+#         if col in ['THTA','THTV',]:
+#             #for moisture
+#             #print('hello:',(new_pro_h[1] - new_pro_h[0])/(listHAGLNEW[4] - listHAGLNEW[3])*(BLH-listHAGLNEW[3]))
+#             #print('hello:',new_pro_h[1] , new_pro_h[0],listHAGLNEW[4] , listHAGLNEW[3],BLH,listHAGLNEW[3])
+#             if len(listHAGLNEW) > 4:
+#                 #print(type(new_pro_h[1]),type(new_pro_h[0]),type(listHAGLNEW[4]),type(listHAGLNEW[3]),type(BLH),type(meanabl))
+#                 dtheta_pre = (new_pro_h[1] - new_pro_h[0])/(listHAGLNEW[4] - listHAGLNEW[3])*(BLH-listHAGLNEW[3]) + new_pro_h[0] - meanabl 
+#                 dtheta = np.max((0.1,dtheta_pre))
+#                 #meanabl = meanabl - (dtheta - dtheta_pre)
+#                 #print('dtheta_pre',dtheta_pre)
+#                 #print('dtheta',dtheta)
+#                 #print('meanabl',meanabl)
+#                 #stop
+#                 
+#             else:
+#                 dtheta = np.nan
+#         else:
+#             if len(listHAGLNEW) > 4:
+#                 #for moisture (it can have both negative and positive slope)
+#                 dtheta = ((new_pro_h[1] - new_pro_h[0])/(listHAGLNEW[4] - listHAGLNEW[3])*(BLH-listHAGLNEW[3]) + new_pro_h[0] - meanabl ) 
+#             else:
+#                 dtheta = np.nan
+#         #print('dtheta',dtheta)
+#         
+#         new_pro = np.array([meanabl,meanabl,meanabl+dtheta]+new_pro_h,dtype=np.float)
+#     
+#         
+#         ONE_COLUMNNEW[-1].insert(len(ONE_COLUMNNEW[-1].columns),col,new_pro)
+#         
+#     #QABSM = np.nanmean(QABS[HAGL <= BLH])
+#     #QABSNEW = np.array([QABSM,QABSM]+list(QABS[HAGL > BLH]))
+#     #ONE_COLUMNNEW.append(pd.DataFrame(zip(HAGLNEW,THTVNEW,QABSNEW),columns=('HAGL','THTV','QABS')))
+#     
+# # we just make a copy of the fields, so that it can be read correctly by CLASS 
+# for dataonecolumn in ONE_COLUMNNEW+[ONE_COLUMN]:
+#     dataonecolumn.insert(len(dataonecolumn.columns),'p_pro',np.array(dataonecolumn.PRES,dtype=np.float)*100.)
+#     dataonecolumn.insert(len(dataonecolumn.columns),'z_pro',np.array(dataonecolumn.HAGL,dtype=np.float))
+#     dataonecolumn.insert(len(dataonecolumn.columns),'theta_pro',np.array(dataonecolumn.THTA,dtype=np.float))
+#     dataonecolumn.insert(len(dataonecolumn.columns),'thetav_pro',np.array(dataonecolumn.THTV,dtype=np.float))
+#     dataonecolumn.insert(len(dataonecolumn.columns),'q_pro',np.array(dataonecolumn.QABS,dtype=np.float))
+#     
+#     angle_x = (90.-np.array(dataonecolumn.DRCT,dtype=np.float))/180.*np.pi # assuming that wind in direction of the south is 0 degrees.
+#     spd = 0.51444* np.array(dataonecolumn.SKNT,dtype=np.float)
+# 
+#     dataonecolumn.insert(len(dataonecolumn.columns),'u_pro',spd * np.sin(angle_x))
+#     dataonecolumn.insert(len(dataonecolumn.columns),'v_pro',spd * np.cos(angle_x))
+# 
+# 
+# # assign fields adopted by CLASS
+# if self.mode == 'o': #original 
+#     PARAMS.insert(0,'h',   np.float(self.h))
+# elif self.mode == 'b':
+#     PARAMS.insert(0,'h',   np.float(self.h))
+# elif self.mode == 'u':
+#     PARAMS.insert(0,'h',   self.h_u)
+# elif self.mode == 'd':
+#     PARAMS.insert(0,'h',   self.h_d)
+# else:
+#     PARAMS.insert(0,'h',   self.h)
+#     
+# 
+# try:
+#     PARAMS.insert(0,'lat', np.float(PARAMS['Station latitude'][0]))
+#     PARAMS.insert(0,'latitude', np.float(PARAMS['Station latitude'][0]))
+# except:
+#     print("could not convert latitude coordinate")
+#     PARAMS.insert(0,'latitude', np.nan)
+#     PARAMS.insert(0,'lat', np.nan)
+# try:
+#     PARAMS.insert(0,'longitude', np.float(PARAMS['Station longitude'][0]))
+#     # we set the actual input parameter value of lon to zero as we are working in local time (as if we were in Greenwhich) 
+#     PARAMS.insert(0,'lon', 0.)
+# except:
+#     print("could not convert longitude coordinate")
+#     PARAMS.insert(0,'longitude', np.nan)
+#     PARAMS.insert(0,'lon', 0.)
+# 
+# if latitude is not None:
+#     print('overwriting latitude with specified value')
+#     PARAMS['latitude'] = np.float(latitude)
+#     PARAMS['lat'] = np.float(latitude)
+# if longitude is not None:
+#     print('overwriting longitude with specified value')
+#     PARAMS['longitude'] = np.float(longitude)
+# try:
+#     #this is the local suntime datetime from which we calculate the hour of the day (assuming we would be in greenwhich hence taking lon=0)
+#     PARAMS['ldatetime'] = PARAMS.datetime.value + dt.timedelta(hours=PARAMS.longitude.value/360.*24.) 
+#     PARAMS['SolarAltitude'] = Pysolar.GetAltitude(PARAMS.lat.value,PARAMS.longitude.value,PARAMS.datetime.value)
+#     PARAMS['SolarAzimuth'] = Pysolar.GetAzimuth(PARAMS.lat.value,PARAMS.longitude.value,PARAMS.datetime.value)
+#     PARAMS['lSunrise'], PARAMS['lSunset'] = Pysolar.util.GetSunriseSunset(PARAMS.lat.value,0.,PARAMS.datetime.value,0.)
+#     # This is the nearest datetime when sun is up (for class)
+#     PARAMS['ldatetime_daylight'] = np.min(np.max(PARAMS['ldatetime'].value ,PARAMS['lSunrise'].value),PARAMS['lSunset'].value) 
+#     # apply the same time shift for UTC datetime
+#     PARAMS['datetime_daylight'] = PARAMS.datetime.value  + (PARAMS.ldatetime_daylight.value  - PARAMS.ldatetime.value)
+#     
+# except:
+#     print("could not get local times for profile, perhaps because of wrong longitude or latitude in the profile description")
+#     PARAMS['ldatetime'] = dt.datetime(1900,1,1)
+#     PARAMS['SolarAltitude'] = np.nan #Pysolar.GetAltitude(PARAMS.lat.value,PARAMS.lon.value,PARAMS.datetime.value)
+#     PARAMS['SolarAzimuth'] = np.nan #Pysolar.GetAzimuth(PARAMS.lat.value,PARAMS.lon.value,PARAMS.datetime.value)
+#     PARAMS['lSunrise'], PARAMS['lSunset'] = dt.datetime(1900,1,1), dt.datetime(1900,1,1) #Pysolar.util.GetSunriseSunset(PARAMS.lat.value,0.,PARAMS.datetime.value,0.)
+#     PARAMS['ldatetime_daylight'] =PARAMS['ldatetime'].value
+#     PARAMS['datetime_daylight'] =PARAMS['datetime'].value
+# 
+# 
+# 
+# PARAMS.insert(0,'day', PARAMS['ldatetime'][0].day)
+# # as we are forcing lon equal to zero this is is expressed in local suntime
+# PARAMS.insert(0,'tstart', PARAMS['ldatetime_daylight'][0].hour + PARAMS['ldatetime_daylight'][0].minute/60. + PARAMS['ldatetime_daylight'][0].second/3600.)
+# 
+#    
+# ONE_COLUMNb = ONE_COLUMNNEW[0]
+# ONE_COLUMNu = ONE_COLUMNNEW[1]
+# ONE_COLUMNd = ONE_COLUMNNEW[2]
+# 
+# 
+# THTVM = np.nanmean(THTV[HAGL <= self.h])
+# PARAMS.insert(len(PARAMS.columns),'THTVM',THTVM)
+# 
+# QABSM = np.nanmean(QABS[HAGL <= self.h])
+# PARAMS.insert(len(PARAMS.columns),'QABSM',QABSM)
+# 
+# PARAMS.insert(len(PARAMS.columns),'self.h',self.h)
+# PARAMS.insert(len(PARAMS.columns),'self.h_u',self.h_u)
+# PARAMS.insert(len(PARAMS.columns),'self.h_d',self.h_d)  
+# 
+# self.he = abs(self.h - self.h_u)
+# self.he = max(self.he,abs(self.h - self.h_d))
+# 
+# #PARAMS.insert(0,'dq',0.)
+# 
+# PARAMS.insert(len(PARAMS.columns),'self.he',self.he)  
+# PARAMS.insert(0,'Ps',np.array(ONE_COLUMN.PRES,dtype='float')[0]*100.)
+# #PARAMS.insert(len(PARAMS.columns),'STNM',STNM)
+# #PARAMS.insert(len(PARAMS.columns),'PATH',webpage)
+# 
+# if self.mode == 'o': #original 
+#     USE_ONECOLUMN = ONE_COLUMN
+#     BLCOLUMN = ONE_COLUMNb # this var is used for investigating whether the original profile is of sufficient quality to be used for analysis or class model input.
+# elif self.mode == 'b': # best BLH
+#     USE_ONECOLUMN = ONE_COLUMNb
+#     BLCOLUMN = ONE_COLUMNb
+# elif self.mode == 'u': # best BLH
+#     USE_ONECOLUMN = ONE_COLUMNu
+#     BLCOLUMN = ONE_COLUMNu
+# elif self.mode == 'd': # best BLH
+#     USE_ONECOLUMN = ONE_COLUMNd
+#     BLCOLUMN = ONE_COLUMNd
+# else:
+#     USE_ONECOLUMN = ONE_COLUMN
+#     BLCOLUMN = ONE_COLUMNb
+# 
+# lt6000 = (BLCOLUMN['HAGL'] < 6000.)
+# lt2500 = (BLCOLUMN['HAGL'] < 2500. + self.h)
+# # print(BLCOLUMN['HAGL'][lt6000])
+# # print(BLCOLUMN['HAGL'][lt2500])
+# # 
+# # print(len(np.where(lt2500)[0]) > 9.) # distance between two points (lower than 2500m) should be smaller than 400 meters
+# 
+# print(BLCOLUMN['HAGL'][lt2500])
+# PARAMS.insert(0,'OK',
+#               ((self.he < 200.) and 
+#                ( len(np.where(lt6000)[0]) > 5) and
+#                (np.array(BLCOLUMN['HAGL'])[-1] >= 6000.) and # the last coordinate had a height higher than 5000.
+#                (not len(np.where(pd.isnull(BLCOLUMN['THTA'][lt6000]))[0]) >0 ) and
+#                (len(np.where(lt2500)[0]) > 10.) and # distance between two points (lower than 2500m) should be smaller than 400 meters
+#                (not len(np.where(pd.isnull(BLCOLUMN['SKNT'][lt6000]))[0]) >0 ) and
+#                (not len(np.where(pd.isnull(BLCOLUMN['DRCT'][lt6000]))[0]) >0 ) and
+#                (not len(np.where(pd.isnull(BLCOLUMN['PRES'][lt6000]))[0]) >0 ) and
+#                (not len(np.where(pd.isnull(BLCOLUMN['QABS'][lt6000]))[0]) >0 ) and
+#                (not (len(np.where(np.array(BLCOLUMN['THTA'][lt6000])[2:] <= np.array(BLCOLUMN['THTA'][lt6000])[1:-1])[0]) >0) ) #absolute increasing
+#               )
+#              )
+# 
+# PARAMS.insert(0,'theta',np.float(list(BLCOLUMN['THTA'])[1]))
+# PARAMS.insert(0,'q',np.float(list(BLCOLUMN['QABS'])[1]))
+# PARAMS.insert(0,'u',np.float(list(BLCOLUMN['u_pro'])[1]))  
+# PARAMS.insert(0,'v',np.float(list(BLCOLUMN['v_pro'])[1]))
+# PARAMS.insert(0,'dtheta',np.float(list(BLCOLUMN['THTA'])[2]-list(BLCOLUMN['THTA'])[1]))
+# PARAMS.insert(0,'dq',np.float(list(BLCOLUMN['QABS'])[2]-list(BLCOLUMN['QABS'])[1]))
+# PARAMS.insert(0,'du',np.float(list(BLCOLUMN['u_pro'])[2]-list(BLCOLUMN['u_pro'])[1]))
+# PARAMS.insert(0,'dv',np.float(list(BLCOLUMN['v_pro'])[2]-list(BLCOLUMN['v_pro'])[1]))
+# 
+# 
+# PARAMS = PARAMS.T
+# 
+# 
+# self.PARAMS = PARAMS
+# self.ONE_COLUMN = USE_ONECOLUMN
+# # if self.mode == 'o': #original 
+# #     self.ONE_COLUMN = ONE_COLUMN
+# # elif self.mode == 'b': # best BLH
+# #     self.ONE_COLUMN = ONE_COLUMNb
+# # elif self.mode == 'u':# upper BLH
+# #     self.ONE_COLUMN = ONE_COLUMNu
+# # elif self.mode == 'd': # lower BLH
+# #     self.ONE_COLUMN=ONE_COLUMNd
+# # else:
+# #     self.ONE_COLUMN = ONE_COLUMN
+
diff --git a/class4gl/era_advection.py b/class4gl/era_advection.py
new file mode 100644
index 0000000..a6c8e8d
--- /dev/null
+++ b/class4gl/era_advection.py
@@ -0,0 +1,160 @@
+
+
+    self.get_idx_in_dataset(globaldata)
+    xrin = globaldata['t'].page['t']
+
+    # 3. prepare artificial xarray input datasets which will allow to make gradient calculations along the W-E directions with xarray on the fly with just one rule
+    # 3.1 we make 'left' and 'right' datasets which will be substracted for calculating gradients
+    # dataset of ilon = 1,2,3...
+    xrleft = xrin.sel(lon=xrin.lon[:-25])
+    # dataset of ilon = 0,1,2...
+    xrright = xrin.sel(lon=xrin.lon[25:])
+    
+    # 3.2 The output will be on a staggered grid with the lon coordinate to the half-level calculated just hereafer in 3.3. 
+    #     Still, we will need to full-level longitude values in the xarray dataset for calculating the grid spacing for the gradients.
+    xrright['flon'] = (('lon',), xrright.lon.values)
+    xrleft['flon'] = (('lon',), xrleft.lon.values)
+    
+    
+    # 3.3 In order to make xarray doing the calculation advection correctly, the 'left' and 'right' values that we need on each grid cell requires equal underlying longitude coordinate values. 
+    xrleft.lon.values = (xrleft.lon.values+xrright.lon.values)/2.
+    xrright.lon.values = xrleft.lon.values
+
+
+    # 4. We do similar preparations for S-N direction. Please note that the advection results for S-N and W-E direction are on different grids, that are also different from the original grid.
+    xrbottom = xrin.sel(lat=xrin.lat[:-25])
+    xrtop = xrin.sel(lat=xrin.lat[25:])
+    xrtop['flat'] = (('lat',), xrtop.lat.values)
+    xrbottom['flat'] = (('lat',), xrbottom.lat.values)
+    xrbottom.lat.values = (xrbottom.lat.values+xrtop.lat.values)/2.
+    xrtop.lat.values = xrbottom.lat.values
+    
+    
+    dia_earth = 40000000.
+
+    # for input variables (COSMO naming)
+    VARS_COSMO = ['QV','U','V','T']
+    # for output variables (ECMWF naming)
+    vars_ECMWF = ['q','u','v','t']
+
+
+    # some netcdf polishing: add units and description to netcdf output. 
+    units = dict(
+          advq_x='kg kg-1 s-1', advq_y='kg kg-1 s-1',
+          advt_x='K s-1',       advt_y='K s-1',
+          advu_x='m s-2',       advu_y='m s-2',
+          advv_x='m s-2',       advv_y='m s-2',
+          divU_x='s-1',         divU_y='s-1',
+                )
+    long_names = dict(
+          advq_x='zonal advection of specific humidity',        
+          advt_x='zonal advection of heat',                     
+          advu_x='zonal advection of zonal wind component',     
+          advv_x='zonal advection of meridional wind component',
+          divU_x='horizontal wind divergence in the zonal direction',
+          advq_y='meridional advection of specific humidity',
+          advt_y='meridional advection of heat',                            
+          advu_y='meridional advection of zonal wind component',            
+          advv_y='meridional advection of meridional wind component',
+          divU_y='horizontal wind divergence in the meridional direction',
+                )
+    #print((xrtop.flat - xrbottom.flat)/360.*dia_earth)
+    # 5. loop over each variable
+
+    # make the selections
+    xrleft_sel = xrleft.isel(time=itimes,lat=ilats,lon=ilons)
+    xrright_sel = xrright.isel(time=itimes,lat=ilats,lon=ilons)
+    xrtop_sel = xrtop.isel(time=itimes,lat=ilats,lon=ilons)
+    xrbottom_sel = xrbottom.isel(time=itimes,lat=ilats,lon=ilons)
+
+    for ivar,var in enumerate(vars_ECMWF):
+        VAR = VARS_COSMO[ivar]
+    
+
+        dims = globaldata.datasets[key].page[key].dims
+        namesmean = list(dims)
+        namesmean.remove('lev')
+        idxmean = [dims.index(namemean) for namemean in namesmean]
+        # over which dimensions we take a mean:
+        dims = globaldata.datasets[key].page[key].dims
+        namesmean = list(dims)
+        namesmean.remove('lev')
+        idxmean = [dims.index(namemean) for namemean in namesmean]
+        #6. actual calculation for the W-E direction
+        #######################################################
+        print('calculation of advection')
+        #######################################################
+
+        if var == 't':
+            self.update(source='era-interim_calc',pars={'adv'+var+'_x':\
+
+
+        ( - (xrright_sel.p**(Rdcp) * xrright_sel.u*xrright[VAR] -
+             xrleft_sel.p**(Rdcp) * xrleft_sel.u*rleft[VAR]) /\
+                ((xrright.flon - xrleft.flon) /360.*dia_earth *np.cos(xrright.lat/180.*np.pi)) /\
+                ((xrright.p**(Rdcp)+xrleft.p**(Rdcp))/2.)
+
+
+
+        self.update(source='era-interim_calc',pars={'adv'+var+'_x':\
+                               (- (xrright_sel.u*xrright_sel[var] - 
+                                   xrleft_sel.u * xrleft_sel[var]) 
+                                  /\
+                                  ((xrright_sel.flon - xrleft_sel.flon) /360.*dia_earth 
+                                   *np.cos(xrright_sel.lat/180.*np.pi))).mean(axis=tuple(idxmean)).values *1.\
+        self.update(source='era-interim_calc',pars={'adv'+var+'_x':\
+                                +\
+                               (- (xrtop_sel.u*xrtop_sel[var] - 
+                                   xrbottom_sel.u * xrbottom_sel[var]) 
+                                  /\
+                                  ((xrtop_sel.flon - xrbottom_sel.flon)\
+                                   /360.*dia_earth).mean(axis=tuple(idxmean)).values *1.
+                                                   })
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+          # over which dimensions we take a mean:
+          dims = globaldata.datasets[key].page[key].dims
+          namesmean = list(dims)
+          namesmean.remove('lev')
+          idxmean = [dims.index(namemean) for namemean in namesmean]
+          
+          value = \
+          globaldata.datasets[key].page[key].isel(time=itimes,
+                                                  lat=ilats,lon=ilons).mean(axis=tuple(idxmean)).values * 1.
+
+          # Ideally, source should be equal to the datakey of globaldata.library 
+          # or globaldata.datasets (eg., DSMW, IGBP-DIS, ERA-INTERIM etc.) 
+          #  but therefore the globaldata class requires a revision to make this work
+          self.update(source='globaldata',air_ac=pd.DataFrame({key:list(value)})) 
+
diff --git a/class4gl/interface/interface.py b/class4gl/interface/interface.py
new file mode 100644
index 0000000..7ededbd
--- /dev/null
+++ b/class4gl/interface/interface.py
@@ -0,0 +1,559 @@
+
+import numpy as np
+
+import pandas as pd
+import sys
+
+import matplotlib
+matplotlib.use('TkAgg')
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--experiments')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--load_globaldata',default=False)
+parser.add_argument('--make_figures',default=None)
+parser.add_argument('--show_control_parameters',default=True)
+parser.add_argument('--figure_filename',default=None)
+parser.add_argument('--figure_filename_2',default=None)
+parser.add_argument('--experiments_labels',default=None)
+parser.add_argument('--tendencies_revised',default=False)
+parser.add_argument('--obs_filter',default='True')
+args = parser.parse_args()
+
+print('Adding python library:',args.c4gl_path_lib)
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import c4gl_interface_soundings,get_record_yaml
+from class4gl import class4gl_input, data_global,class4gl,units
+#from sklearn.metrics import mean_squared_error
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+#import seaborn.apionly as sns
+import pylab as pl
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import kde
+from scipy.stats import pearsonr                                                
+from taylorDiagram import TaylorDiagram
+from matplotlib import ticker
+# import importlib
+# importlib.reload(mpl); importlib.reload(plt); importlib.reload(sns)
+
+
+
+
+def abline(slope, intercept,axis):
+    """Plot a line from slope and intercept"""
+    #axis = plt.gca()
+    x_vals = np.array(axis.get_xlim())
+    y_vals = intercept + slope * x_vals
+    axis.plot(x_vals, y_vals, 'k--')
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+if args.experiments_labels is None:
+    keylabels = args.experiments.strip().split(' ')
+else:
+    keylabels = args.experiments_labels.strip().split(';')
+
+
+
+# EXPS  =\
+# {
+# 'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_ADV':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'IOPS_ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+# }
+
+if bool(args.load_globaldata):
+    # iniitialize global data
+    globaldata = data_global()
+    # ...  and load initial data pages
+    globaldata.load_datasets(recalc=0)
+else:
+    globaldata = None
+
+c4gldata = {}
+for key in args.experiments.strip(' ').split(' '):
+    
+    c4gldata[key] = c4gl_interface_soundings( \
+                      args.path_experiments+'/'+key+'/',\
+                      args.path_forcing,\
+                      globaldata,\
+                      refetch_records=False,
+                      tendencies_revised = args.tendencies_revised,
+                      obs_filter = (args.obs_filter == 'True')
+                    )
+
+if args.make_figures:
+    # the lines below activate TaylorPlots but it is disabled for now
+    fig = plt.figure(figsize=(10,7))   #width,height
+    i = 1                                                                           
+    axes = {}         
+    axes_taylor = {}         
+    
+    colors = ['r','g','b','m','y','c']
+    symbols = ['*','x','+','o']
+    dias = {}
+    
+    varkeys = ['h','theta','q']
+    for varkey in varkeys:                                                    
+        axes[varkey] = fig.add_subplot(2,3,i)                                       
+        #axes_taylor[varkey] = fig.add_subplot(2,3,i+3)                                       
+    
+        #print(obs.std())
+        dias[varkey] =  TaylorDiagram(1., srange=[0.0,1.7],fig=fig, rect=(230+i+3),label='Reference')
+        if i == 0:
+            dias[varkey]._ax.axis["left"].label.set_text(\
+                "Standard deviation (model) / Standard deviation (observations)")
+            # dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+            # dias[varkey]._ax.axis["left"].axis.set_major_locator(np.arange(0.,2.,0.25))
+        #dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+        # Q95 = obs.quantile(0.95)
+        # Q95 = obs.quantile(0.90)
+        # Add RMS contours, and label them
+        contours = dias[varkey].add_contours(levels=5, colors='0.5') # 5 levels
+        dias[varkey].ax.clabel(contours, inline=1, fontsize=10, fmt='%.1f')
+        #dia._ax.set_title(season.capitalize())
+    
+        dias[varkey].add_grid()
+    
+    
+        #dia.ax.plot(x99,y99,color='k')
+    
+        
+        for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+            # cc = c4gldata[key].frames['stats']['records_all_stations_ini']['cc']
+            # clearsky = (cc < 0.05)
+            # mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats'].loc[clearsky]['d'+varkey+'dt']
+            # obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats'].loc[clearsky]['d'+varkey+'dt']
+            mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt']
+            obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt']
+            x, y = obs.values,mod.values
+            print(key,len(obs.values))
+    
+            STD_OBS = obs.std()
+            #scores
+            PR = pearsonr(mod,obs)[0]
+            RMSE = rmse(obs,mod)                                               
+            BIAS = np.mean(mod) - np.mean(obs)
+            STD = mod.std()
+            
+            # fit = np.polyfit(x,y,deg=1)
+            # axes[varkey].plot(x, fit[0] * x + fit[1],\
+            #                   color=colors[ikey],alpha=0.8,lw=2,\
+            #                   label=key+", "+\
+            #                               'R = '+str(round(PR,3))+', '+\
+            #                               'RMSE = '+str(round(RMSE,5))+units['d'+varkey+'dt']+', '+\
+            #                               'BIAS = '+str(round(BIAS,5))+units['d'+varkey+'dt'] )
+            # axes[varkey].legend(fontsize=5)
+            
+            # print(STD)
+            # print(PR)
+            dias[varkey].add_sample(STD/STD_OBS, PR,
+                           marker='o', ms=5, ls='',
+                           #mfc='k', mec='k', # B&W
+                           mfc=colors[ikey], mec=colors[ikey], # Colors
+                           label=key)
+    
+        # put ticker position, see
+        # https://matplotlib.org/examples/ticks_and_spines/tick-locators.html 
+        # dia.ax.axis['bottom'].
+        # dia.ax.axis['left'].
+        # dia.ax.axis['left'].
+    
+        i += 1
+    
+    i = 0
+    for varkey in ['h','theta','q']:                                                    
+        ikey = 0
+        key = list(args.experiments.strip().split(' '))[ikey]
+        # cc = c4gldata[key].frames['stats']['records_all_stations_ini']['cc']
+        # clearsky = (cc < 0.05)
+    
+        # mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats'].loc[clearsky]['d'+varkey+'dt']
+        # obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats'].loc[clearsky]['d'+varkey+'dt']
+        mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt']
+        obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt']
+    
+    
+        nbins=40       
+        x, y = obs.values,mod.values
+        
+        xi, yi = np.mgrid[x.min():x.max():nbins*1j, y.min():y.max():nbins*1j]
+        zi = np.zeros_like(xi)*np.nan       
+        for ibin in range(nbins):
+            xmin = x.min() + ibin * (x.max() - x.min())/nbins
+            xmax = xmin + (x.max() - x.min())/nbins
+            in_bin = ((x >= xmin) & (x < xmax))
+            ybin = y[in_bin]
+            xbin = x[in_bin]
+            if len(ybin) > 20:
+                k = kde.gaussian_kde((ybin))
+                zi[ibin] = k(np.vstack([yi[ibin].flatten()]))
+        zi = zi/np.sum(zi,axis=1)[:,np.newaxis]
+        zi_int = zi.cumsum(axis=1) 
+                     #  label=key+", "+\
+                     #                    'R = '+str(round(PR[0],3))+', '+\
+                     #                    'RMSE = '+str(round(RMSE,5))+', '+\
+                     #                    'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+        axes[varkey].contour(xi, yi, zi_int.reshape(xi.shape),levels=[0.16,0.5,0.84] ,
+                colors=['darkred','lightgreen','darkred'],linewidths=[1,2,1])
+        axes[varkey].contourf(xi, yi, zi_int.reshape(xi.shape),levels=[0.16,0.84] ,
+                colors=['darkred'],alpha=0.5,)
+        nanxi = xi[zi != np.nan]
+        axes[varkey].set_xlim((nanxi.min(),nanxi.max()))
+        axes[varkey].set_ylim((nanxi.min(),nanxi.max()))
+        print(varkey,(nanxi.min(),nanxi.max()))
+    
+    
+        latex = {}
+        latex['dthetadt'] =  r'$d \theta / dt $'
+        latex['dqdt'] =      r'$d q / dt $'
+        latex['dhdt'] =      r'$d h / dt $'
+    
+        axes[varkey].set_xlabel('observations')     
+        axes[varkey].set_title(latex['d'+varkey+'dt']+' ['+units['d'+varkey+'dt']+']')                                     
+    
+        PR = pearsonr(mod,obs)[0]
+        RMSE = rmse(obs,mod)                                               
+        BIAS = np.mean(mod) - np.mean(obs)
+        STD = mod.std()
+    
+        axes[varkey].scatter(obs,mod, label='(only) '+key+", "+\
+                                      'R = '+str(round(PR,3))+', '+\
+                                      'RMSE = '+str(round(RMSE,5))+units['d'+varkey+'dt']+', '+\
+                                      'BIAS = '+str(round(BIAS,5))+units['d'+varkey+'dt'] ,\
+                             s=0.1,alpha=0.14,color='k')
+        axes[varkey].legend(fontsize=5)
+        
+
+
+
+        axes[varkey].set_xlabel('observations')     
+        if i==0:                                    
+            axes[varkey].set_ylabel('model')                                            
+        abline(1,0,axis=axes[varkey])
+        i +=1
+    
+    
+    
+    # legend for different forcing simulations (colors)
+    ax = fig.add_axes([0.05,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    leg = []
+    for ikey,key in enumerate(args.experiments.strip().split(' ')):
+        leg1, = ax.plot([],colors[ikey]+'o' ,markersize=10)
+        leg.append(leg1)
+    ax.axis('off')
+    #leg1 =
+    ax.legend(leg,list(args.experiments.strip().split(' ')),loc=2,fontsize=10)
+    
+    
+    # # legend for different stations (symbols)
+    # ax = fig.add_axes([0.25,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    # leg = []
+    # isymbol = 0
+    # for icurrent_station,current_station in c4gldata[key].frames['worldmap']['stations'].table.iterrows():
+    #     leg1, = ax.plot([],'k'+symbols[isymbol] ,markersize=10)
+    #     leg.append(leg1)
+    #     isymbol += 1
+    # 
+    # # symbol for all stations
+    # leg1, = ax.plot([],'ko',markersize=10)
+    # leg.append(leg1)
+    
+    
+    # ax.axis('off')
+    # ax.legend(leg,['HUMPPA','BLLAST','GOAMAZON','All'],loc=2,fontsize=10)
+    
+    
+    fig.subplots_adjust(top=0.95,bottom=0.20,left=0.08,right=0.94,hspace=0.28,wspace=0.29)
+    
+    
+    #pl.legend(leglist,('EMI:WOC','EMI:MED','EMI:BEC'),loc=2,fontsize=16,prop={'family':
+    # figfn = '/user/data/gent/gvo000/gvo00090/D2D/archive/report/global_eval_report_cs.png'
+    # fig.savefig(figfn,dpi=200); print("Image file written to:", figfn)
+    
+    if args.figure_filename is not None:
+        fig.savefig(args.figure_filename,dpi=200); print("Image file written to:",args.figure_filename)
+    fig.show()  
+
+    if bool(args.show_control_parameters):
+
+        import seaborn as sns
+
+        pkmn_type_colors = [
+                                            '#A0A0A0',  # Poison
+                                            '#78C850',  # Grass
+                                            '#F08030',  # Fire
+                                            '#6890F0',  # Water
+                                            '#F08030',  # Fire
+                                            '#C03028',  # Fighting
+                                            '#F85888',  # Psychic
+                                            '#A8B820',  # Bug
+                                            '#A8A878',  # Normal
+                                            '#F8D030',  # Electric
+                                            '#E0C068',  # Ground
+                                            '#EE99AC',  # Fairy
+                                            '#B8A038',  # Rock
+                                            '#705898',  # Ghost
+                                            '#98D8D8',  # Ice
+                                            '#7038F8',  # Dragon
+                                           ]
+
+
+
+        sns.set_style('whitegrid')
+        #sns.set()
+        fig = pl.figure(figsize=(11,7))
+        i = 1
+        axes = {}
+        data_all = pd.DataFrame()
+        data_input = pd.DataFrame()
+        
+        
+        
+        # #for varkey in ['theta','q']:     
+        # EF =\
+        #     c4gldata[key].frames['stats']['records_all_stations_ini'].BR/(1.+\
+        #     c4gldata[key].frames['stats']['records_all_stations_ini'].BR)
+        # EF[EF<0] = np.nan
+        # EF[EF>1] = np.nan
+        
+        # c4gldata[key].frames['stats']['records_all_stations_ini']['EF'] = EF
+        
+        ikey = 0
+        key = list(args.experiments.strip().split(' '))[ikey]
+        data_all = pd.DataFrame()
+
+        tempdatamodstats = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_end_obs_stats'].copy())
+        tempdatamodstats["source"] = "Soundings"
+        tempdatamodstats["source_index"] = "Soundings"
+
+        ini_ref = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_ini'].copy())
+        tempdataini_this = pd.DataFrame(ini_ref.copy())
+
+        tempdatamodstats['dates']= tempdataini_this.ldatetime.dt.date
+        tempdatamodstats['STNID']= tempdataini_this.STNID
+        tempdatamodstats['source']= "Soundings"
+        tempdatamodstats['source_index']= "Soundings"
+        tempdatamodstats.set_index(['source_index','STNID','dates'],inplace=True)
+        #print('hello')
+
+        tempdataini = pd.DataFrame(ini_ref)
+        tempdataini["source"] = "Soundings"
+        tempdataini["source_index"] = "Soundings"
+        tempdataini = tempdataini.set_index(['source_index','STNID','dates'])
+        #print('hello2')
+
+
+        data_all = pd.concat([data_all,tempdatamodstats],axis=0)
+        data_input = pd.concat([data_input,tempdataini],axis=0)
+        #print(data_input.shape)
+        #print(data_all.shape)
+
+            
+        for ikey,key in enumerate(list(args.experiments.strip().split(' '))):
+            keylabel = keylabels[ikey]
+
+            tempdatamodstats = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_end_mod_stats'].copy())
+            tempdataini_this= pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_ini'].copy())
+            tempdatamodstats['dates']= tempdataini_this.ldatetime.dt.date
+            tempdatamodstats['STNID']= tempdataini_this.STNID
+            tempdatamodstats['source']= keylabel
+            tempdatamodstats['source_index']= keylabel
+            tempdatamodstats.set_index(['source_index','STNID','dates'],inplace=True)
+            #print('hello')
+
+
+            tempdataini = pd.DataFrame(ini_ref.copy())
+            tempdataini["source"] = keylabel
+            tempdataini["source_index"] = keylabel
+            tempdataini = tempdataini.set_index(['source_index','STNID','dates'])
+    
+
+            #print('hello2')
+            index_intersect = tempdataini.index.intersection(tempdatamodstats.index)
+            #print('hello3')
+
+            tempdataini = tempdataini.loc[index_intersect]
+            #print('hello4')
+            tempdatamodstats = tempdatamodstats.loc[index_intersect]
+            #print('hello5')
+
+
+            # data[varkey] = tempdatamodstats['d'+varkey+'dt']
+            data_all = pd.concat([data_all,tempdatamodstats],axis=0)
+            data_input = pd.concat([data_input, tempdataini],axis=0)
+            #print(data_input.shape)
+            #print(data_all.shape)
+
+        data_input.cc = data_input.cc.clip(0.,+np.inf)
+
+        for varkey in ['h','theta','q']:
+            varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+            data_all = data_all.rename(columns={'d'+varkey+'dt':varkey_full})
+            
+        data_input['advt_tropo'] = data_input['advt_tropo'] * 3600.
+        data_all['advt_tropo'] = data_input['advt_tropo']
+            #print(data_input.shape)
+            #print(data_all.shape)
+        #print('hello6')
+        #print(data_all.columns)
+        #print('hello7')
+        i = 1
+        for varkey in ['h','theta','q']:
+            input_keys =['wg','advt_tropo']
+            for input_key in input_keys:
+                varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+
+                #print('hello8')
+                #print(data_input.shape)
+                #print(data_all.shape)
+                units['advt_tropo'] = 'K/h'
+                input_key_full = input_key + "["+units[input_key]+"]"
+                data_all[input_key_full] = pd.cut(x=data_input[input_key].values,bins=8,precision=2)
+                data_input[input_key_full] = pd.cut(x=data_input[input_key].values,bins=8,precision=2,)
+                #print('hello9')
+                #print(data_input.shape)
+                #print(data_all.shape)
+                
+                qvalmax = data_all[varkey_full].quantile(0.999)
+                qvalmin = data_all[varkey_full].quantile(0.001)
+                select_data = (data_all[varkey_full] >= qvalmin) & (data_all[varkey_full] < qvalmax)
+                #print('hello11')
+                data_all = data_all[select_data]
+                #print('hello12')
+                data_input = data_input[select_data.values]
+                #print('hello13')
+                #print(data_input.shape)
+                #print(data_all.shape)
+                #print('hello10')
+                
+                sns.set(style="ticks", palette="pastel")
+                ax = fig.add_subplot(3,len(input_keys),i)
+                #sns.violinplot(x=input_key_full,y=varkey_full,data=data_all,hue='source',linewidth=2.,palette="muted",split=True,inner='quart') #,label=key+", R = "+str(round(PR[0],3)),data=data)       
+                
+                #ax.set_title(input_key_full)
+                sb = sns.boxplot(x=input_key_full, y=varkey_full, hue="source",
+                                 palette=pkmn_type_colors,
+                                # palette=["m", "g",'r','b'],
+                                 linewidth=1.2, data=data_all,sym='')
+                if i ==1:
+                     plt.legend(loc='upper right',fontsize=7.)
+                else:
+                     ax.get_legend().set_visible(False)
+                #     plt.legend('off')
+                if i >= 5:
+                    #ax.set_xticklabels(labels=['['+str(i)+','+str(i+1)+'[' for i in list(range(0,7))]+['[7,8]'])
+
+                    ax.set_xticklabels(labels=ax.get_xticklabels(),rotation=45.,ha='right')
+                else:
+                    ax.set_xticklabels([])
+                    ax.set_xlabel('')
+
+                if np.mod(i,len(input_keys)) != 0:
+                    ax.set_yticklabels([])
+                    ax.set_ylabel('')
+
+                if varkey == 'q':
+                    ticks = ticker.FuncFormatter(lambda x, pos:
+                                                 '{0:g}'.format(x*1000.))
+                    #ax.xaxis.set_major_formatter(ticks)
+                    ax.yaxis.set_major_formatter(ticks)
+
+                    ax.set_ylabel(latex['d'+varkey+'dt']+' ['+r'$10^{-3} \times $'+units['d'+varkey+'dt']+']')        
+                else:
+                    ax.set_ylabel(latex['d'+varkey+'dt']+' ['+units['d'+varkey+'dt']+']')        
+
+
+                for j,artist in enumerate(ax.artists):
+                    if np.mod(j,len(list(args.experiments.strip().split(' ')))+1) !=0:
+                        # Set the linecolor on the artist to the facecolor, and set the facecolor to None
+                        #print(j,artist)
+                        col = artist.get_facecolor()
+                        #print(j,artist)
+                        artist.set_edgecolor(col)
+                        #print(j,artist)
+                        artist.set_facecolor('None')
+                
+                        # Each box has 6 associated Line2D objects (to make the whiskers, fliers, etc.)
+                        # Loop over them here, and use the same colour as above
+                        
+                        for k in range(j*5,j*5+5):
+                            line = ax.lines[k]
+                            line.set_color(col)
+                            line.set_mfc(col)
+                            line.set_mec(col)
+                
+                # Also fix the legend
+                j = 0
+                for legpatch in ax.get_legend().get_patches():
+                    if j > 0:
+
+                        col = legpatch.get_facecolor()
+                        legpatch.set_edgecolor(col)
+                        legpatch.set_facecolor('None')
+                    j +=1
+
+
+
+
+                #ax.grid()
+                #sns.despine(offset=10, trim=True)
+                i +=1
+        fig.tight_layout()
+        fig.subplots_adjust( bottom=0.12,left=0.15,top=0.99,right=0.99,wspace=0.05,hspace=0.05,)
+        if args.figure_filename_2 is not None:
+            fig.savefig(args.figure_filename_2,dpi=200); print("Image file written to:", args.figure_filename_2)
+        fig.show()
+
+
+
diff --git a/class4gl/interface/interface_cloudiness.py b/class4gl/interface/interface_cloudiness.py
new file mode 100644
index 0000000..82afd1e
--- /dev/null
+++ b/class4gl/interface/interface_cloudiness.py
@@ -0,0 +1,550 @@
+'''
+import numpy as np
+
+import pandas as pd
+import sys
+
+import matplotlib
+matplotlib.use('TkAgg')
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--experiments')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--load_globaldata',default=False)
+parser.add_argument('--make_figures',default=None)
+parser.add_argument('--show_control_parameters',default=True)
+parser.add_argument('--figure_filename',default=None)
+parser.add_argument('--figure_filename_2',default=None)
+parser.add_argument('--experiments_labels',default=None)
+args = parser.parse_args()
+
+print('Adding python library:',args.c4gl_path_lib)
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import c4gl_interface_soundings,get_record_yaml
+from class4gl import class4gl_input, data_global,class4gl,units
+#from sklearn.metrics import mean_squared_error
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+#import seaborn.apionly as sns
+import pylab as pl
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import kde
+from scipy.stats import pearsonr                                                
+from taylorDiagram import TaylorDiagram
+from matplotlib import ticker
+# import importlib
+# importlib.reload(mpl); importlib.reload(plt); importlib.reload(sns)
+
+
+
+if args.experiments_labels is None:
+    keylabels = args.experiments.strip().split(' ')
+else:
+    keylabels = args.experiments_labels.strip().split(';')
+
+def abline(slope, intercept,axis):
+    """Plot a line from slope and intercept"""
+    #axis = plt.gca()
+    x_vals = np.array(axis.get_xlim())
+    y_vals = intercept + slope * x_vals
+    axis.plot(x_vals, y_vals, 'k--')
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+
+
+
+# EXPS  =\
+# {
+# 'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_ADV':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'IOPS_ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+# }
+
+if bool(args.load_globaldata):
+    # iniitialize global data
+    globaldata = data_global()
+    # ...  and load initial data pages
+    globaldata.load_datasets(recalc=0)
+else:
+    globaldata = None
+
+c4gldata = {}
+for key in args.experiments.strip(' ').split(' '):
+    
+    c4gldata[key] = c4gl_interface_soundings( \
+                      args.path_experiments+'/'+key+'/',\
+                      args.path_forcing,\
+                      globaldata,\
+                      refetch_records=False
+                    )
+    
+'''
+if args.make_figures:
+    # the lines below activate TaylorPlots but it is disabled for now
+    fig = plt.figure(figsize=(10,7))   #width,height
+    i = 1                                                                           
+    axes = {}         
+    axes_taylor = {}         
+    
+    colors = ['r','g','b','m']
+    symbols = ['*','x','+']
+    dias = {}
+    
+    for varkey in ['h','theta','q']:                                                    
+        axes[varkey] = fig.add_subplot(2,3,i)                                       
+        #axes_taylor[varkey] = fig.add_subplot(2,3,i+3)                                       
+    
+        #print(obs.std())
+        dias[varkey] =  TaylorDiagram(1., srange=[0.0,1.7],fig=fig, rect=(230+i+3),label='Reference')
+        if i == 0:
+            dias[varkey]._ax.axis["left"].label.set_text(\
+                "Standard deviation (model) / Standard deviation (observations)")
+            # dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+            # dias[varkey]._ax.axis["left"].axis.set_major_locator(np.arange(0.,2.,0.25))
+        #dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+        # Q95 = obs.quantile(0.95)
+        # Q95 = obs.quantile(0.90)
+        # Add RMS contours, and label them
+        contours = dias[varkey].add_contours(levels=5, colors='0.5') # 5 levels
+        dias[varkey].ax.clabel(contours, inline=1, fontsize=10, fmt='%.1f')
+        #dia._ax.set_title(season.capitalize())
+    
+        dias[varkey].add_grid()
+    
+    
+        #dia.ax.plot(x99,y99,color='k')
+    
+        
+        for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+            # cc = c4gldata[key].frames['stats']['records_all_stations_ini']['cc']
+            # clearsky = (cc < 0.05)
+            # mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats'].loc[clearsky]['d'+varkey+'dt']
+            # obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats'].loc[clearsky]['d'+varkey+'dt']
+            mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats']['d'+varkey+'dt']
+            obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt']
+            x, y = obs.values,mod.values
+            print(key,len(obs.values))
+    
+            STD_OBS = obs.std()
+            #scores
+            PR = pearsonr(mod,obs)[0]
+            RMSE = rmse(obs,mod)                                               
+            BIAS = np.mean(mod) - np.mean(obs)
+            STD = mod.std()
+            
+            # fit = np.polyfit(x,y,deg=1)
+            # axes[varkey].plot(x, fit[0] * x + fit[1],\
+            #                   color=colors[ikey],alpha=0.8,lw=2,\
+            #                   label=key+", "+\
+            #                               'R = '+str(round(PR,3))+', '+\
+            #                               'RMSE = '+str(round(RMSE,5))+units['d'+varkey+'dt']+', '+\
+            #                               'BIAS = '+str(round(BIAS,5))+units['d'+varkey+'dt'] )
+            # axes[varkey].legend(fontsize=5)
+            
+            # print(STD)
+            # print(PR)
+            dias[varkey].add_sample(STD/STD_OBS, PR,
+                           marker='o', ms=5, ls='',
+                           #mfc='k', mec='k', # B&W
+                           mfc=colors[ikey], mec=colors[ikey], # Colors
+                           label=keylabels[ikey])
+    
+        # put ticker position, see
+        # https://matplotlib.org/examples/ticks_and_spines/tick-locators.html 
+        # dia.ax.axis['bottom'].
+        # dia.ax.axis['left'].
+        # dia.ax.axis['left'].
+    
+        i += 1
+    
+    i = 0
+    for varkey in ['h','theta','q']:                                                    
+        ikey = 0
+        key = list(args.experiments.strip().split(' '))[ikey]
+        keylabel = keylabels[ikey]
+        cc = c4gldata[key].frames['stats']['records_all_stations_ini']['cc']
+        clearsky = (cc < 0.05)
+    
+        mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats'].loc[clearsky]['d'+varkey+'dt']
+        obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats'].loc[clearsky]['d'+varkey+'dt']
+    
+    
+        nbins=40       
+        x, y = obs.values,mod.values
+        
+        xi, yi = np.mgrid[x.min():x.max():nbins*1j, y.min():y.max():nbins*1j]
+        zi = np.zeros_like(xi)*np.nan       
+        for ibin in range(nbins):
+            xmin = x.min() + ibin * (x.max() - x.min())/nbins
+            xmax = xmin + (x.max() - x.min())/nbins
+            in_bin = ((x >= xmin) & (x < xmax))
+            ybin = y[in_bin]
+            xbin = x[in_bin]
+            if len(ybin) > 20:
+                k = kde.gaussian_kde((ybin))
+                zi[ibin] = k(np.vstack([yi[ibin].flatten()]))
+        zi = zi/np.sum(zi,axis=1)[:,np.newaxis]
+        zi_int = zi.cumsum(axis=1) 
+                     #  label=key+", "+\
+                     #                    'R = '+str(round(PR[0],3))+', '+\
+                     #                    'RMSE = '+str(round(RMSE,5))+', '+\
+                     #                    'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+        axes[varkey].contour(xi, yi, zi_int.reshape(xi.shape),levels=[0.16,0.5,0.84] ,
+                colors=['darkred','lightgreen','darkred'],linewidths=[1,2,1])
+        axes[varkey].contourf(xi, yi, zi_int.reshape(xi.shape),levels=[0.16,0.84] ,
+                colors=['darkred'],alpha=0.5,)
+        nanxi = xi[zi != np.nan]
+        axes[varkey].set_xlim((nanxi.min(),nanxi.max()))
+        axes[varkey].set_ylim((nanxi.min(),nanxi.max()))
+        print(varkey,(nanxi.min(),nanxi.max()))
+    
+    
+        latex = {}
+        latex['dthetadt'] =  r'$d \theta / dt $'
+        latex['dqdt'] =      r'$d q / dt $'
+        latex['dhdt'] =      r'$d h / dt $'
+    
+        axes[varkey].set_xlabel('observations')     
+        axes[varkey].set_title(latex['d'+varkey+'dt']+' ['+units['d'+varkey+'dt']+']')                                     
+    
+        PR = pearsonr(mod,obs)[0]
+        RMSE = rmse(obs,mod)                                               
+        BIAS = np.mean(mod) - np.mean(obs)
+        STD = mod.std()
+    
+        axes[varkey].scatter(obs,mod, label='(only) '+keylabel+", "+\
+                                      'R = '+str(round(PR,3))+', '+\
+                                      'RMSE = '+str(round(RMSE,5))+units['d'+varkey+'dt']+', '+\
+                                      'BIAS = '+str(round(BIAS,5))+units['d'+varkey+'dt'] ,\
+                             s=0.1,alpha=0.14,color='k')
+        axes[varkey].legend(fontsize=5)
+        
+
+
+
+        axes[varkey].set_xlabel('observations')     
+        if i==0:                                    
+            axes[varkey].set_ylabel('model')                                            
+        abline(1,0,axis=axes[varkey])
+        i +=1
+    
+    
+    
+    # legend for different forcing simulations (colors)
+    ax = fig.add_axes([0.05,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    leg = []
+    for ikey,key in enumerate(args.experiments.strip().split(' ')):
+        
+        leg1, = ax.plot([],colors[ikey]+'o' ,markersize=10)
+        leg.append(leg1)
+    ax.axis('off')
+    #leg1 =
+    ax.legend(leg,list(args.experiments.strip().split(' ')),loc=2,fontsize=10)
+    
+    
+    # # legend for different stations (symbols)
+    # ax = fig.add_axes([0.25,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    # leg = []
+    # isymbol = 0
+    # for icurrent_station,current_station in c4gldata[key].frames['worldmap']['stations'].table.iterrows():
+    #     leg1, = ax.plot([],'k'+symbols[isymbol] ,markersize=10)
+    #     leg.append(leg1)
+    #     isymbol += 1
+    # 
+    # # symbol for all stations
+    # leg1, = ax.plot([],'ko',markersize=10)
+    # leg.append(leg1)
+    
+    
+    # ax.axis('off')
+    # ax.legend(leg,['HUMPPA','BLLAST','GOAMAZON','All'],loc=2,fontsize=10)
+    
+    
+    fig.subplots_adjust(top=0.95,bottom=0.20,left=0.08,right=0.94,hspace=0.28,wspace=0.29)
+    
+    
+    #pl.legend(leglist,('EMI:WOC','EMI:MED','EMI:BEC'),loc=2,fontsize=16,prop={'family':
+    # figfn = '/user/data/gent/gvo000/gvo00090/D2D/archive/report/global_eval_report_cs.png'
+    # fig.savefig(figfn,dpi=200); print("Image file written to:", figfn)
+    
+    if args.figure_filename is not None:
+        fig.savefig(args.figure_filename,dpi=200); print("Image file written to:",args.figure_filename)
+    fig.show()  
+
+    if bool(args.show_control_parameters):
+
+        import seaborn as sns
+
+        pkmn_type_colors = [
+                                            '#A0A0A0',  # Poison
+                                            '#78C850',  # Grass
+                                            '#F08030',  # Fire
+                                            '#6890F0',  # Water
+                                            '#F08030',  # Fire
+                                            '#C03028',  # Fighting
+                                            '#F85888',  # Psychic
+                                            '#A8B820',  # Bug
+                                            '#A8A878',  # Normal
+                                            '#F8D030',  # Electric
+                                            '#E0C068',  # Ground
+                                            '#EE99AC',  # Fairy
+                                            '#B8A038',  # Rock
+                                            '#705898',  # Ghost
+                                            '#98D8D8',  # Ice
+                                            '#7038F8',  # Dragon
+                                           ]
+
+
+
+        sns.set_style('whitegrid')
+        #sns.set()
+        fig = pl.figure(figsize=(11,7))
+        i = 1
+        axes = {}
+        data_all = pd.DataFrame()
+        data_input = pd.DataFrame()
+        
+        
+        
+        # #for varkey in ['theta','q']:     
+        # EF =\
+        #     c4gldata[key].frames['stats']['records_all_stations_ini'].BR/(1.+\
+        #     c4gldata[key].frames['stats']['records_all_stations_ini'].BR)
+        # EF[EF<0] = np.nan
+        # EF[EF>1] = np.nan
+        
+        # c4gldata[key].frames['stats']['records_all_stations_ini']['EF'] = EF
+        
+        ikey = 0
+        key = list(args.experiments.strip().split(' '))[ikey]
+        data_all = pd.DataFrame()
+
+        tempdatamodstats = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats'].copy())
+        tempdatamodstats["source"] = "Soundings"
+        tempdatamodstats["source_index"] = "Soundings"
+
+        ini_ref = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_ini'].copy())
+        tempdataini_this = pd.DataFrame(ini_ref.copy())
+
+        tempdatamodstats['dates']= tempdataini_this.ldatetime.dt.date
+        tempdatamodstats['STNID']= tempdataini_this.STNID
+        tempdatamodstats['source']= "Soundings"
+        tempdatamodstats['source_index']= "Soundings"
+        tempdatamodstats.set_index(['source_index','STNID','dates'],inplace=True)
+        #print('hello')
+
+        tempdataini = pd.DataFrame(ini_ref)
+        tempdataini["source"] = "Soundings"
+        tempdataini["source_index"] = "Soundings"
+        tempdataini = tempdataini.set_index(['source_index','STNID','dates'])
+        #print('hello2')
+
+
+        data_all = pd.concat([data_all,tempdatamodstats],axis=0)
+        data_input = pd.concat([data_input,tempdataini],axis=0)
+        #print(data_input.shape)
+        #print(data_all.shape)
+
+            
+        for ikey,key in enumerate(list(args.experiments.strip().split(' '))):
+            #keylabels=args.experimts.strip().split(' ')
+            keylabel = keylabels[ikey]
+
+            tempdatamodstats = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_mod_stats'].copy())
+            tempdataini_this= pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_ini'].copy())
+            tempdatamodstats['dates']= tempdataini_this.ldatetime.dt.date
+            tempdatamodstats['STNID']= tempdataini_this.STNID
+            tempdatamodstats['source']= keylabel
+            tempdatamodstats['source_index']= keylabel
+            tempdatamodstats.set_index(['source_index','STNID','dates'],inplace=True)
+            #print('hello')
+
+
+            tempdataini = pd.DataFrame(ini_ref.copy())
+            tempdataini["source"] = keylabel
+            tempdataini["source_index"] = keylabel
+            tempdataini = tempdataini.set_index(['source_index','STNID','dates'])
+    
+
+            #print('hello2')
+            index_intersect = tempdataini.index.intersection(tempdatamodstats.index)
+            #print('hello3')
+
+            tempdataini = tempdataini.loc[index_intersect]
+            #print('hello4')
+            tempdatamodstats = tempdatamodstats.loc[index_intersect]
+            #print('hello5')
+
+
+            # data[varkey] = tempdatamodstats['d'+varkey+'dt']
+            data_all = pd.concat([data_all,tempdatamodstats],axis=0)
+            data_input = pd.concat([data_input, tempdataini],axis=0)
+            #print(data_input.shape)
+            #print(data_all.shape)
+
+        data_input.cc = data_input.cc.clip(0.,+np.inf)
+
+        for varkey in ['h','theta','q']:
+            varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+            data_all = data_all.rename(columns={'d'+varkey+'dt':varkey_full})
+            #print(data_input.shape)
+            #print(data_all.shape)
+        #print('hello6')
+        #print(data_all.columns)
+        #print('hello7')
+        for varkey in ['h','theta','q']:
+            input_keys =['cc']
+            for input_key in input_keys:
+                varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+
+                #print('hello8')
+                #print(data_input.shape)
+                #print(data_all.shape)
+                input_key_full = input_key + "["+units[input_key]+"]"
+                data_all[input_key_full] = pd.cut(x=data_input[input_key].values,bins=8,precision=2)
+                data_input[input_key_full] = pd.cut(x=data_input[input_key].values,bins=8,precision=2,)
+                #print('hello9')
+                #print(data_input.shape)
+                #print(data_all.shape)
+                
+                qvalmax = data_all[varkey_full].quantile(0.999)
+                qvalmin = data_all[varkey_full].quantile(0.001)
+                select_data = (data_all[varkey_full] >= qvalmin) & (data_all[varkey_full] < qvalmax)
+                #print('hello11')
+                data_all = data_all[select_data]
+                #print('hello12')
+                data_input = data_input[select_data.values]
+                #print('hello13')
+                #print(data_input.shape)
+                #print(data_all.shape)
+                #print('hello10')
+                
+                sns.set(style="ticks", palette="pastel")
+                ax = fig.add_subplot(3,len(input_keys),i)
+                #sns.violinplot(x=input_key_full,y=varkey_full,data=data_all,hue='source',linewidth=2.,palette="muted",split=True,inner='quart') #,label=key+", R = "+str(round(PR[0],3)),data=data)       
+                
+                #ax.set_title(input_key_full)
+                sb = sns.boxplot(x=input_key_full, y=varkey_full, hue="source",
+                                 palette=pkmn_type_colors,
+                                # palette=["m", "g",'r','b'],
+                                 linewidth=1.2, data=data_all,sym='')
+                if i ==1:
+                     plt.legend(loc='upper right',fontsize=7.,frameon=True,framealpha=0.7)
+                else:
+                     ax.get_legend().set_visible(False)
+                #     plt.legend('off')
+                if i >= 3:
+                    ax.set_xticklabels(labels=['['+str(i)+','+str(i+1)+'[' for i in list(range(0,7))]+['[7,8]'])
+                    ax.set_xlabel('Cloudiness [okta]')
+                else:
+                    ax.set_xticklabels([])
+                    ax.set_xlabel('')
+
+                if np.mod(i,len(input_keys)) != 0:
+                    ax.set_yticklabels([])
+                    ax.set_ylabel('')
+
+                if varkey == 'q':
+                    ticks = ticker.FuncFormatter(lambda x, pos:
+                                                 '{0:g}'.format(x*1000.))
+                    #ax.xaxis.set_major_formatter(ticks)
+                    ax.yaxis.set_major_formatter(ticks)
+
+                    ax.set_ylabel(latex['d'+varkey+'dt']+' ['+r'$10^{-3} \times $'+units['d'+varkey+'dt']+']')        
+                else:
+                    ax.set_ylabel(latex['d'+varkey+'dt']+' ['+units['d'+varkey+'dt']+']')        
+
+
+                for j,artist in enumerate(ax.artists):
+                    if np.mod(j,len(list(args.experiments.strip().split(' ')))+1) !=0:
+                        # Set the linecolor on the artist to the facecolor, and set the facecolor to None
+                        #print(j,artist)
+                        col = artist.get_facecolor()
+                        #print(j,artist)
+                        artist.set_edgecolor(col)
+                        #print(j,artist)
+                        artist.set_facecolor('None')
+                
+                        # Each box has 6 associated Line2D objects (to make the whiskers, fliers, etc.)
+                        # Loop over them here, and use the same colour as above
+                        
+                        for k in range(j*5,j*5+5):
+                            line = ax.lines[k]
+                            line.set_color(col)
+                            line.set_mfc(col)
+                            line.set_mec(col)
+                
+                # Also fix the legend
+                j = 0
+                for legpatch in ax.get_legend().get_patches():
+                    if j > 0:
+
+                        col = legpatch.get_facecolor()
+                        legpatch.set_edgecolor(col)
+                        legpatch.set_facecolor('None')
+                    j +=1
+
+
+
+
+                #ax.grid()
+                #sns.despine(offset=10, trim=True)
+                i +=1
+        fig.tight_layout()
+        fig.subplots_adjust( bottom=0.12,left=0.15,top=0.99,right=0.99,wspace=0.05,hspace=0.05,)
+        if args.figure_filename_2 is not None:
+            fig.savefig(args.figure_filename_2,dpi=200); print("Image file written to:", args.figure_filename_2)
+        fig.show()
+
+
+
diff --git a/class4gl/interface/interface_koeppen.py b/class4gl/interface/interface_koeppen.py
new file mode 100644
index 0000000..ada9a44
--- /dev/null
+++ b/class4gl/interface/interface_koeppen.py
@@ -0,0 +1,777 @@
+import numpy as np
+
+import pandas as pd
+import sys
+
+import matplotlib
+matplotlib.use('TkAgg')
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--experiments')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--load_globaldata',default=False)
+parser.add_argument('--make_figures',default=None)
+parser.add_argument('--show_control_parameters',default=True)
+parser.add_argument('--figure_filename',default=None)
+parser.add_argument('--figure_filename_2',default=None)
+parser.add_argument('--experiments_labels',default=None)
+parser.add_argument('--tendencies_revised',default=False)
+args = parser.parse_args()
+
+print('Adding python library:',args.c4gl_path_lib)
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import c4gl_interface_soundings,get_record_yaml
+from class4gl import class4gl_input, data_global,class4gl,units
+#from sklearn.metrics import mean_squared_error
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+#import seaborn.apionly as sns
+import pylab as pl
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import kde
+from scipy.stats import pearsonr                                                
+from taylorDiagram import TaylorDiagram
+from matplotlib import ticker
+import xarray as xr
+# import importlib
+# importlib.reload(mpl); importlib.reload(plt); importlib.reload(sns)
+
+
+if args.experiments_labels is None:
+    keylabels = args.experiments.strip().split(' ')
+else:
+    keylabels = args.experiments_labels.strip().split(';')
+
+def abline(slope, intercept,axis):
+    """Plot a line from slope and intercept"""
+    #axis = plt.gca()
+    x_vals = np.array(axis.get_xlim())
+    y_vals = intercept + slope * x_vals
+    axis.plot(x_vals, y_vals, 'k--')
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+
+
+
+# EXPS  =\
+# {
+# 'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_ADV':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'IOPS_ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+# }
+
+if bool(args.load_globaldata):
+    # iniitialize global data
+    globaldata = data_global()
+    # ...  and load initial data pages
+    globaldata.load_datasets(recalc=0)
+else:
+    globaldata = None
+
+c4gldata = {}
+for key in args.experiments.strip(' ').split(' '):
+    
+    c4gldata[key] = c4gl_interface_soundings( \
+                      args.path_experiments+'/'+key+'/',\
+                      args.path_forcing+'/',\
+                      globaldata,\
+                      refetch_records=False,
+                      tendencies_revised = args.tendencies_revised
+                    )
+
+key = args.experiments.strip(' ').split(' ')[0]
+xrkoeppen = xr.open_dataset('/user/data/gent/gvo000/gvo00090/EXT/data/KOEPPEN/Koeppen-Geiger.nc')
+koeppenlookuptable = pd.DataFrame()
+koeppenlookuptable['KGCID'] = pd.Series(xrkoeppen['KGCID'])
+
+
+
+
+KGCID=    ['Af', 'Am', 'As', 'Aw', 'BSh', 'BSk', 'BWh', 'BWk', 'Cfa', 'Cfb','Cfc', 'Csa', 'Csb', 'Csc', 'Cwa','Cwb', 'Cwc', 'Dfa', 'Dfb', 'Dfc','Dfd', 'Dsa', 'Dsb', 'Dsc', 'Dsd','Dwa', 'Dwb', 'Dwc', 'Dwd', 'EF','ET', 'Ocean'] 
+KGCcolors=["#960000", "#FF0000", "#FF6E6E", "#FFCCCC", "#CC8D14", "#CCAA54", "#FFCC00", "#FFFF64", "#007800", "#005000", "#003200", "#96FF00", "#00D700", "#00AA00", "#BEBE00", "#8C8C00", "#5A5A00", "#550055", "#820082", "#C800C8", "#FF6EFF", "#646464", "#8C8C8C", "#BEBEBE", "#E6E6E6", "#6E28B4", "#B464FA", "#C89BFA", "#C8C8FF", "#6496FF", "#64FFFF", "#F5FFFF"]
+
+def brightness(rrggbb):
+    """ W3C brightness definition
+        input:
+            hexadecimal color in the format:
+            #RRGGBB
+        output: value between 0 and 1
+    """
+    print(rrggbb)
+    rr = int(rrggbb[1:3],16)/int('FF',16)
+    gg = int(rrggbb[3:5],16)/int('FF',16)
+    bb = int(rrggbb[5:7],16)/int('FF',16)
+    #rr = math.floor(rrggbb/10000.)
+    #gg = math.floor((rrggbb - rr*10000.)/100.)
+    #bb = rrggbb - rr*10000 - gg*100
+    return (rr * 299. + gg * 587. + bb * 114.) / 1000.
+
+kgccolors = {}
+for iKGC,KGCname in enumerate(KGCID):
+    kgccolors[KGCname] = [KGCcolors[iKGC],'white' if (brightness(KGCcolors[iKGC])<0.5) else 'black']
+
+# kgccolors = {
+#     'Dfa':['navy','white'],
+#     'Cfb':['green','white']       ,
+#     'BSk':['tan','black']      ,
+#     'Csb':['lightgreen','black'] ,     
+#     'Cfa':['darkgreen','white']  ,    
+#     'BWh':['orange','black']      ,
+#     'Aw' :['pink','black'],
+#     'Dwc':['rebeccapurple','white'] ,    
+#     'Dfb':['darkviolet','white']    , 
+# }
+kgcnames = {
+    'Dfa':'snow \n fully humid \n hot summer',
+    'Cfb':'green'       ,
+    'BSk':'4'      ,
+    'Csb':'5'      ,
+    'Cfa':'darkgreen' ,     
+    'BWh':'6'      ,
+    'Aw' :'7'     ,
+    'Dwc':'8'     ,
+    'Dfb':'9'     ,
+    #'Dfa':'',
+}
+for KGCID in list(pd.Series(xrkoeppen['KGCID'])):
+    if KGCID not in kgcnames.keys():
+        kgcnames[KGCID] = KGCID
+    if KGCID not in kgccolors.keys():
+        kgccolors[KGCID] = ['k','k']
+
+
+koeppenlookuptable['color'] = ""
+koeppenlookuptable['textcolor'] = ""
+koeppenlookuptable['name'] = ""
+for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+    print(ikoeppen)
+    print(koeppen.KGCID)
+    print(kgccolors[koeppen.KGCID])
+    koeppenlookuptable['color'].loc[ikoeppen] = kgccolors[koeppen.KGCID][0]
+    koeppenlookuptable['textcolor'].loc[ikoeppen] = kgccolors[koeppen.KGCID][1]
+    koeppenlookuptable['name'].loc[ikoeppen] = kgcnames[koeppen.KGCID]
+
+
+
+c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname'] =  \
+    c4gldata[key].frames['stats']['records_all_stations_ini']['KGC'].map(koeppenlookuptable['KGCID'])
+
+print('sort the climate classes according to the amount ')
+koeppenlookuptable['amount'] = ""
+
+exclude_koeppen = ['Dfc','Cwb']
+
+for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+
+    if koeppen['KGCID'] not in exclude_koeppen:
+        print(ikoeppen,':',koeppen)
+        kgc_select = (c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname'] == koeppen['KGCID'])
+        print(np.sum(kgc_select))
+        koeppenlookuptable.iloc[ikoeppen]['amount'] = np.sum(kgc_select)
+    else:
+        koeppenlookuptable.iloc[ikoeppen]['amount'] = 0
+
+koeppenlookuptable = koeppenlookuptable.sort_values('amount',ascending=False)
+koeppenlookuptable = koeppenlookuptable[:9]
+koeppenlookuptable = koeppenlookuptable.sort_index()
+
+
+if args.make_figures:
+    # the lines below activate TaylorPlots but it is disabled for now
+    fig = plt.figure(figsize=(11,7))   #width,height
+    i = 1                                                                           
+    axes = {}         
+    axes_taylor = {}         
+    
+    colors = ['r','g','b','m','y','purple','orange','sienna','navy']
+    symbols = ['*','x','+']
+    dias = {}
+
+
+
+    i = 1
+    for varkey in ['h','theta','q']:                                                    
+        dias[varkey] =  TaylorDiagram(1., srange=[0.0,1.7],fig=fig, rect=(230+i+3),label='Reference')
+        axes[varkey] = fig.add_subplot(2,3,i)                                       
+
+        for ikey,key in enumerate(args.experiments.strip(' ').split(' ')[:1]):
+            icolor = 0
+            for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+                print(ikoeppen,':',koeppen)
+                kgc_select = (c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname'] == koeppen['KGCID'])
+                
+                koeppen_mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats']['d'+varkey+'dt'][kgc_select]
+                koeppen_obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt'][kgc_select]
+    
+                #axes[varkey].scatter(koeppen_obs,koeppen_mod,marker=symbols[ikoeppen],color=colors[ikey])
+                         #  label=key+", "+\
+                         #                    'R = '+str(round(PR[0],3))+', '+\
+                         #                    'RMSE = '+str(round(RMSE,5))+', '+\
+                         #                    'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+    
+    
+    
+            # # pl.scatter(obs,mod,label=key+", "+\
+            # #                              'R = '+str(round(PR[0],3))+', '+\
+            # #                              'RMSE = '+str(round(RMSE,5))+', '+\
+            # #                              'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+                
+                print('hellobla')
+                print(koeppen.KGCID)
+                print(koeppen.color)
+                dias[varkey].add_sample(koeppen_mod.std()/koeppen_obs.std(),
+                               pearsonr(koeppen_mod,koeppen_obs)[0],
+                               annotate=koeppen.KGCID, color=koeppen.textcolor,weight='bold',fontsize=5.,\
+                               bbox={'edgecolor':'black','boxstyle':'circle','fc':koeppen.color,'alpha':0.7}
+                               )
+                icolor += 1
+    
+            latex = {}
+            latex['dthetadt'] =  r'$d \theta / dt $'
+            latex['dqdt'] =      r'$d q / dt $'
+            latex['dhdt'] =      r'$d h / dt $'
+    
+            axes[varkey].set_xlabel('observations')     
+            axes[varkey].set_title(latex['d'+varkey+'dt']+' ['+units['d'+varkey+'dt']+']')                                     
+        if i==1:                                    
+            axes[varkey].set_ylabel('model')                                            
+        abline(1,0,axis=axes[varkey])
+        i +=1
+
+    
+    i = 0
+    for varkey in ['h','theta','q']:                                                    
+        #axes_taylor[varkey] = fig.add_subplot(2,3,i+3)                                       
+    
+        #print(obs.std())
+        if i == 1:
+            dias[varkey]._ax.axis["left"].label.set_text(\
+                "standard deviation (model) / standard deviation (observations)")
+            # dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+            # dias[varkey]._ax.axis["left"].axis.set_major_locator(np.arange(0.,2.,0.25))
+        #dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+        # Q95 = obs.quantile(0.95)
+        # Q95 = obs.quantile(0.90)
+        # Add RMS contours, and label them
+        contours = dias[varkey].add_contours(levels=5, colors='0.5') # 5 levels
+        dias[varkey].ax.clabel(contours, inline=1, fontsize=10, fmt='%.1f')
+        #dia._ax.set_title(season.capitalize())
+    
+        dias[varkey].add_grid()
+    
+    
+        #dia.ax.plot(x99,y99,color='k')
+    
+        
+        #for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+        for ikey,key in enumerate(args.experiments.strip(' ').split(' ')[:1]):
+            # cc = c4gldata[key].frames['stats']['records_all_stations_ini']['cc']
+            # clearsky = (cc < 0.05)
+            # mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats'].loc[clearsky]['d'+varkey+'dt']
+            # obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats'].loc[clearsky]['d'+varkey+'dt']
+            mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats']['d'+varkey+'dt']
+            obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt']
+
+            print ('filtering classes (showing bad performance)', exclude_koeppen,' from results!')
+            filter_classes = ~(c4gldata[key].frames['stats']['records_all_stations_ini'].KGCname.isin(exclude_koeppen))
+            mod = mod.loc[filter_classes]
+            obs = obs.loc[filter_classes]
+            x, y = obs.values,mod.values
+            print(key,len(obs.values))
+    
+            STD_OBS = obs.std()
+            #scores
+            PR = pearsonr(mod,obs)[0]
+            RMSE = rmse(obs,mod)                                               
+            BIAS = np.mean(mod) - np.mean(obs)
+            STD = mod.std()
+            
+            # fit = np.polyfit(x,y,deg=1)
+            # axes[varkey].plot(x, fit[0] * x + fit[1],\
+            #                   color=colors[ikey],alpha=0.8,lw=2,\
+            #                   label=key+", "+\
+            #                               'R = '+str(round(PR,3))+', '+\
+            #                               'RMSE = '+str(round(RMSE,5))+units['d'+varkey+'dt']+', '+\
+            #                               'BIAS = '+str(round(BIAS,5))+units['d'+varkey+'dt'] )
+            # axes[varkey].legend(fontsize=5)
+            
+            # print(STD)
+            # print(PR)
+            dias[varkey].add_sample(STD/STD_OBS, PR,\
+                               annotate='All', zorder=100,color='black',weight='bold',fontsize=5.,\
+                                    bbox={'edgecolor':'black','boxstyle':'circle','fc':'lightgrey','alpha':0.6}\
+                            )
+    
+        # put ticker position, see
+        # https://matplotlib.org/examples/ticks_and_spines/tick-locators.html 
+        # dia.ax.axis['bottom'].
+        # dia.ax.axis['left'].
+        # dia.ax.axis['left'].
+    
+        i += 1
+
+    
+    i = 0
+    for varkey in ['h','theta','q']:                                                    
+        ikey = 0
+        key = list(args.experiments.strip().split(' '))[ikey]
+        keylabel = keylabels[ikey]
+        # cc = c4gldata[key].frames['stats']['records_all_stations_ini']['cc']
+        # clearsky = (cc < 0.05)
+    
+        # mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats'].loc[clearsky]['d'+varkey+'dt']
+        # obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats'].loc[clearsky]['d'+varkey+'dt']
+    
+        mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats']['d'+varkey+'dt']
+        obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt']
+        print ('filtering classes (showing bad performance)', exclude_koeppen,' from results!')
+        filter_classess = ~(c4gldata[key].frames['stats']['records_all_stations_ini'].KGCname.isin(exclude_koeppen))
+        mod = mod.loc[filter_classes]
+        obs = obs.loc[filter_classes]
+    
+        nbins=40       
+        x, y = obs.values,mod.values
+        
+        xi, yi = np.mgrid[x.min():x.max():nbins*1j, y.min():y.max():nbins*1j]
+        zi = np.zeros_like(xi)*np.nan       
+        for ibin in range(nbins):
+            xmin = x.min() + ibin * (x.max() - x.min())/nbins
+            xmax = xmin + (x.max() - x.min())/nbins
+            in_bin = ((x >= xmin) & (x < xmax))
+            ybin = y[in_bin]
+            xbin = x[in_bin]
+            if len(ybin) > 20:
+                k = kde.gaussian_kde((ybin))
+                zi[ibin] = k(np.vstack([yi[ibin].flatten()]))
+        zi = zi/np.sum(zi,axis=1)[:,np.newaxis]
+        zi_int = zi.cumsum(axis=1) 
+                     #  label=key+", "+\
+                     #                    'R = '+str(round(PR[0],3))+', '+\
+                     #                    'RMSE = '+str(round(RMSE,5))+', '+\
+                     #                    'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+        axes[varkey].contour(xi, yi, zi_int.reshape(xi.shape),levels=[0.16,0.5,0.84] ,
+                colors=['darkred','lightgreen','darkred'],linewidths=[1,2,1])
+        axes[varkey].contourf(xi, yi, zi_int.reshape(xi.shape),levels=[0.16,0.84] ,
+                colors=['darkred'],alpha=0.5,)
+        nanxi = xi[zi != np.nan]
+        axes[varkey].set_xlim((nanxi.min(),nanxi.max()))
+        axes[varkey].set_ylim((nanxi.min(),nanxi.max()))
+        print(varkey,(nanxi.min(),nanxi.max()))
+    
+    
+        latex = {}
+        latex['dthetadt'] =  r'$d \theta / dt $'
+        latex['dqdt'] =      r'$d q / dt $'
+        latex['dhdt'] =      r'$d h / dt $'
+    
+        axes[varkey].set_xlabel('observations')     
+
+        if varkey == 'q':
+            axes[varkey].set_title(latex['d'+varkey+'dt']+' ['+r'$10^{-3} \times $'+units['d'+varkey+'dt']+']')        
+        else:
+            axes[varkey].set_title(latex['d'+varkey+'dt']+' ['+units['d'+varkey+'dt']+']')     
+       #  c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname']
+
+        PR = pearsonr(mod,obs)[0]
+        RMSE = rmse(obs,mod)                                               
+        BIAS = np.mean(mod) - np.mean(obs)
+        STD = mod.std()
+    
+        axes[varkey].scatter(obs,mod, label='All',s=0.1,alpha=0.14,color='k')
+
+
+
+        #axes[varkey].legend(fontsize=5)
+
+        #trans = ax.get_xaxis_transform() # x in data untis, y in axes fraction
+
+        if varkey == 'q':
+            annotate_text = 'PC = '+str(round(PR,3))+'\n'+\
+                           'RMSE = '+str(round(RMSE*1000.,5))+r'$10^{-3} \times $'+ units['d'+varkey+'dt']+'\n'+\
+                           'BIAS = '+str(round(BIAS*1000.,5))+r'$10^{-3} \times $'+units['d'+varkey+'dt'] 
+        else:
+            annotate_text = 'PC = '+str(round(PR,3))+'\n'+\
+                           'RMSE = '+str(round(RMSE,5))+units['d'+varkey+'dt']+'\n'+\
+                           'BIAS = '+str(round(BIAS,5))+units['d'+varkey+'dt'] 
+
+
+        ann = axes[varkey].annotate(annotate_text, xy=(0.05, .95 ), xycoords='axes fraction',fontsize=8,
+       horizontalalignment='left', verticalalignment='top' 
+        )
+
+
+        axes[varkey].set_xlabel('observations')     
+        if i==0:                                    
+            axes[varkey].set_ylabel('model')                                            
+        abline(1,0,axis=axes[varkey])
+        i +=1
+
+        #axes[varkey].axis('equal')
+        axes[varkey].set_aspect('equal')
+
+        # To specify the number of ticks on both or any single axes
+        # plt.locator_params(axis='x', nbins=6)
+        #plt.locator_params( nbins=10)
+        axes[varkey].xaxis.set_major_locator(ticker.MaxNLocator(4))
+        axes[varkey].yaxis.set_major_locator(ticker.MaxNLocator(4))
+        # axes[varkey].xaxis.set_major_locator(ticker.MultipleLocator(5))
+        # axes[varkey].yaxis.set_major_locator(ticker.MultipleLocator(5))
+
+        if varkey == 'q':
+            ticks = ticker.FuncFormatter(lambda x, pos:
+                                         '{0:g}'.format(x*1000.))
+            axes[varkey].xaxis.set_major_formatter(ticks)
+            axes[varkey].yaxis.set_major_formatter(ticks)
+
+        #     # axes[varkey].set_xticklabels(labels=ax.get_xticklabels()*1000.)
+        #     # axes[varkey].set_yticklabels(labels=ax.get_yticklabels()*1000.)
+    
+    
+    # # legend for different forcing simulations (colors)
+    # ax = fig.add_axes([0.05,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    # leg = []
+    # for ikey,key in enumerate(args.experiments.strip().split(' ')):
+    #     leg1, = ax.plot([],colors[ikey]+'o' ,markersize=10)
+    #     leg.append(leg1)
+    # ax.axis('off')
+    # #leg1 =
+    # ax.legend(leg,list(args.experiments.strip().split(' ')),loc=2,fontsize=10)
+    
+    
+    # # legend for different stations (symbols)
+    # ax = fig.add_axes([0.25,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    # leg = []
+    # isymbol = 0
+    # for icurrent_station,current_station in c4gldata[key].frames['worldmap']['stations'].table.iterrows():
+    #     leg1, = ax.plot([],'k'+symbols[isymbol] ,markersize=10)
+    #     leg.append(leg1)
+    #     isymbol += 1
+    # 
+    # # symbol for all stations
+    # leg1, = ax.plot([],'ko',markersize=10)
+    # leg.append(leg1)
+    7
+    
+    # ax.axis('off')
+    # ax.legend(leg,['HUMPPA','BLLAST','GOAMAZON','All'],loc=2,fontsize=10)
+    
+    
+    fig.subplots_adjust(top=0.95,bottom=0.09,left=0.08,right=0.94,hspace=0.35,wspace=0.29)
+    
+    
+    #pl.legend(leglist,('EMI:WOC','EMI:MED','EMI:BEC'),loc=2,fontsize=16,prop={'family':
+    # figfn = '/user/data/gent/gvo000/gvo00090/D2D/archive/report/global_eval_report_cs.png'
+    # fig.savefig(figfn,dpi=200); print("Image file written to:", figfn)
+    
+    if args.figure_filename is not None:
+        fig.savefig(args.figure_filename,dpi=200); print("Image file written to:",args.figure_filename)
+    fig.show()  
+
+    if bool(args.show_control_parameters):
+
+        import seaborn as sns
+
+        pkmn_type_colors = [
+                                            '#A0A0A0',  # Poison
+                                            '#78C850',  # Grass
+                                            '#F08030',  # Fire
+                                            '#6890F0',  # Water
+                                            '#F08030',  # Fire
+                                            '#C03028',  # Fighting
+                                            '#F85888',  # Psychic
+                                            '#A8B820',  # Bug
+                                            '#A8A878',  # Normal
+                                            '#F8D030',  # Electric
+                                            '#E0C068',  # Ground
+                                            '#EE99AC',  # Fairy
+                                            '#B8A038',  # Rock
+                                            '#705898',  # Ghost
+                                            '#98D8D8',  # Ice
+                                            '#7038F8',  # Dragon
+                                           ]
+
+
+
+        sns.set_style('whitegrid')
+        #sns.set()
+        fig = pl.figure(figsize=(11,7))
+        i = 1
+        axes = {}
+        data_all = pd.DataFrame()
+        data_input = pd.DataFrame()
+        
+        
+        
+        # #for varkey in ['theta','q']:     
+        # EF =\
+        #     c4gldata[key].frames['stats']['records_all_stations_ini'].BR/(1.+\
+        #     c4gldata[key].frames['stats']['records_all_stations_ini'].BR)
+        # EF[EF<0] = np.nan
+        # EF[EF>1] = np.nan
+        
+        # c4gldata[key].frames['stats']['records_all_stations_ini']['EF'] = EF
+        
+        ikey = 0
+        key = list(args.experiments.strip().split(' '))[ikey]
+        data_all = pd.DataFrame()
+
+        tempdatamodstats = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats'].copy())
+        
+
+        tempdatamodstats["source"] = "soundings"
+        tempdatamodstats["source_index"] = "soundings"
+
+        ini_ref = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_ini'].copy())
+        tempdataini_this = pd.DataFrame(ini_ref.copy())
+
+        tempdatamodstats['dates']= tempdataini_this.ldatetime.dt.date
+        tempdatamodstats['STNID']= tempdataini_this.STNID
+        tempdatamodstats['source']= "soundings"
+        tempdatamodstats['source_index']= "soundings"
+        tempdatamodstats.set_index(['source_index','STNID','dates'],inplace=True)
+        #print('hello')
+
+        tempdataini = pd.DataFrame(ini_ref)
+        tempdataini["source"] = "soundings"
+        tempdataini["source_index"] = "soundings"
+        tempdataini = tempdataini.set_index(['source_index','STNID','dates'])
+        #print('hello2')
+
+
+        data_all = pd.concat([data_all,tempdatamodstats],axis=0)
+        data_input = pd.concat([data_input,tempdataini],axis=0)
+        #print(data_input.shape)
+        #print(data_all.shape)
+
+            
+        for ikey,key in enumerate(list(args.experiments.strip().split(' '))):
+            keylabel = keylabels[ikey]
+
+            tempdatamodstats = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_mod_stats'].copy())
+            tempdataini_this= pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_ini'].copy())
+
+            tempdatamodstats['dates']= tempdataini_this.ldatetime.dt.date
+            tempdatamodstats['STNID']= tempdataini_this.STNID
+            tempdatamodstats['source']= keylabel
+            tempdatamodstats['source_index']= keylabel
+            tempdatamodstats.set_index(['source_index','STNID','dates'],inplace=True)
+            #print('hello')
+
+
+            tempdataini = pd.DataFrame(ini_ref.copy())
+            tempdataini["source"] = keylabel
+            tempdataini["source_index"] = keylabel
+            tempdataini = tempdataini.set_index(['source_index','STNID','dates'])
+    
+
+            #print('hello2')
+            index_intersect = tempdataini.index.intersection(tempdatamodstats.index)
+            #print('hello3')
+
+            tempdataini = tempdataini.loc[index_intersect]
+            #print('hello4')
+            tempdatamodstats = tempdatamodstats.loc[index_intersect]
+            #print('hello5')
+
+
+            print ('filtering classes (showing bad performance)', exclude_koeppen,' from results!')
+            # data[varkey] = tempdatamodstats['d'+varkey+'dt']
+            data_all = pd.concat([data_all,tempdatamodstats],axis=0)
+            data_input = pd.concat([data_input, tempdataini],axis=0)
+            #print(data_input.shape)
+            #print(data_all.shape)
+
+        data_input.cc = data_input.cc.clip(0.,+np.inf)
+
+        for varkey in ['h','theta','q']:
+            varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+            data_all = data_all.rename(columns={'d'+varkey+'dt':varkey_full})
+            data_all['KGCname'] = data_input['KGCname']
+
+
+
+
+            #print(data_input.shape)
+            #print(data_all.shape)
+        # xrkoeppen = xr.open_dataset('/user/data/gent/gvo000/gvo00090/EXT/data/KOEPPEN/Koeppen-Geiger.nc')
+        # lookuptable = pd.Series(xrkoeppen['KGCID'])
+        # data_all['KGCname'] = data_input['KGC'].map(lookuptable)
+        #print('hello6')
+        #print(data_all.columns)
+        #print('hello7')
+        for varkey in ['h','theta','q']:
+            #input_keys =['wg','cc']
+            #for input_key in input_keys:
+            varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+
+            #print('hello8')
+            #print(data_input.shape)
+            #print(data_all.shape)
+            #input_key_full = input_key + "["+units[input_key]+"]"
+            #print('hello9')
+            #print(data_input.shape)
+            #print(data_all.shape)
+            print ('Excluding extreme values from the classes plots')
+            qvalmax = data_all[varkey_full].quantile(0.999)
+            qvalmin = data_all[varkey_full].quantile(0.001)
+            select_data = (data_all[varkey_full] >= qvalmin) & (data_all[varkey_full] < qvalmax)
+            #print('hello11')
+            data_all = data_all[select_data]
+            #print('hello12')
+            data_input = data_input[select_data.values]
+
+            data_input = data_input[data_all.KGCname.isin(list(koeppenlookuptable.KGCID))]
+            data_all = data_all[data_all.KGCname.isin(list(koeppenlookuptable.KGCID))]
+            #print('hello13')
+            #print(data_input.shape)
+            #print(data_all.shape)
+            #print('hello10')
+            
+
+            sns.set(style="ticks", palette="pastel")
+            ax = fig.add_subplot(3,1,i)
+            #sns.violinplot(x='KGC',y=varkey_full,data=data_all,hue='source',linewidth=2.,palette="muted",split=True,inner='quart') #,label=key+", R = "+str(round(PR[0],3)),data=data)       
+            
+            #ax.set_title(input_key_full)
+            sb = sns.boxplot(x='KGCname', y=varkey_full, hue="source",
+                             palette=pkmn_type_colors,
+                            # palette=["m", "g",'r','b'],
+                             linewidth=1.2, data=data_all,sym='')
+            if i ==1:
+                 plt.legend(loc='upper right',fontsize=7.,frameon=True,framealpha=0.7)
+            else:
+                 ax.get_legend().set_visible(False)
+            #     plt.legend('off')
+            if i >= 3:
+                idx = 0
+                for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+
+                    ax.annotate(koeppen.KGCID,
+                                xy=((idx+.5)/len(koeppenlookuptable),-0.00),
+                                color=koeppen.textcolor, 
+                                xycoords='axes fraction',
+                                weight='bold',
+                                fontsize=8.,
+                                horizontalalignment='center',
+                                verticalalignment='center' ,
+                                bbox={'edgecolor':'black',
+                                      'boxstyle':'circle',
+                                      'fc':koeppen.color,
+                                      'alpha':1.0}
+                               )
+                    idx+=1
+                ax.set_xticklabels([])#labels=ax.get_xticklabels())
+                ax.set_xlabel('Köppen climate class')
+            else:
+                ax.set_xticklabels([])
+                ax.set_xlabel('')
+
+            # ax.set_yticklabels([])
+            # ax.set_ylabel('')
+            if varkey == 'q':
+                ticks = ticker.FuncFormatter(lambda x, pos:
+                                             '{0:g}'.format(x*1000.))
+                #ax.xaxis.set_major_formatter(ticks)
+                ax.yaxis.set_major_formatter(ticks)
+
+                ax.set_ylabel(latex['d'+varkey+'dt']+' ['+r'$10^{-3} \times $'+units['d'+varkey+'dt']+']')        
+            else:
+                ax.set_ylabel(latex['d'+varkey+'dt']+' ['+units['d'+varkey+'dt']+']')        
+
+
+
+
+            for j,artist in enumerate(ax.artists):
+                if np.mod(j,len(list(args.experiments.strip().split(' ')))+1) !=0:
+                    # Set the linecolor on the artist to the facecolor, and set the facecolor to None
+                    #print(j,artist)
+                    col = artist.get_facecolor()
+                    #print(j,artist)
+                    artist.set_edgecolor(col)
+                    #print(j,artist)
+                    artist.set_facecolor('None')
+            
+                    # Each box has 6 associated Line2D objects (to make the whiskers, fliers, etc.)
+                    # Loop over them here, and use the same colour as above
+                    
+                    for k in range(j*5,j*5+5):
+                        line = ax.lines[k]
+                        line.set_color(col)
+                        line.set_mfc(col)
+                        line.set_mec(col)
+            
+            # Also fix the legend
+            j = 0
+            for legpatch in ax.get_legend().get_patches():
+                if j > 0:
+
+                    col = legpatch.get_facecolor()
+                    legpatch.set_edgecolor(col)
+                    legpatch.set_facecolor('None')
+                j +=1
+
+
+
+
+
+            #ax.grid()
+            #sns.despine(offset=10, trim=True)
+            i +=1
+        fig.tight_layout()
+        fig.subplots_adjust( bottom=0.12,left=0.15,top=0.99,right=0.99,wspace=0.05,hspace=0.05,)
+        if args.figure_filename_2 is not None:
+            fig.savefig(args.figure_filename_2,dpi=200); print("Image file written to:", args.figure_filename_2)
+        fig.show()
+
+
+
diff --git a/class4gl/interface/interface_new_koeppen.py b/class4gl/interface/interface_new_koeppen.py
new file mode 100644
index 0000000..3dbe77f
--- /dev/null
+++ b/class4gl/interface/interface_new_koeppen.py
@@ -0,0 +1,1155 @@
+import numpy as np
+import pandas as pd
+import sys
+import matplotlib
+matplotlib.use('TkAgg')
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--experiments')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--load_globaldata',default=False)
+parser.add_argument('--make_figures',default=None)
+parser.add_argument('--show_control_parameters',default=True)
+parser.add_argument('--figure_filename',default=None)
+parser.add_argument('--figure_filename_2',default=None)
+parser.add_argument('--experiments_labels',default=None)
+parser.add_argument('--obs_filter',default='True')
+args = parser.parse_args()
+
+print('Adding python library:',args.c4gl_path_lib)
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import c4gl_interface_soundings,get_record_yaml
+from class4gl import class4gl_input, data_global,class4gl,units
+#from sklearn.metrics import mean_squared_error
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+import seaborn.apionly as sns
+import pylab as pl
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import kde
+from scipy.stats import pearsonr                                                
+from taylorDiagram import TaylorDiagram
+from matplotlib import ticker
+import xarray as xr
+# import importlib
+# importlib.reload(mpl); importlib.reload(plt); importlib.reload(sns)
+
+
+if args.experiments_labels is None:
+    keylabels = args.experiments.strip().split(' ')
+else:
+    keylabels = args.experiments_labels.strip().split(';')
+
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+
+
+
+# EXPS  =\
+# {
+# 'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_ADV':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'IOPS_ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+# }
+
+if bool(args.load_globaldata):
+    # iniitialize global data
+    globaldata = data_global()
+    # ...  and load initial data pages
+    globaldata.load_datasets(recalc=0)
+else:
+    globaldata = None
+
+c4gldata = {}
+for key in args.experiments.strip(' ').split(' '):
+    
+    c4gldata[key] = c4gl_interface_soundings( \
+                      args.path_experiments+'/'+key+'/',\
+                      args.path_forcing+'/',\
+                      globaldata,\
+                      refetch_records=False,
+                      obs_filter = (args.obs_filter == 'True')
+                                            
+                    )
+
+sns.reset_orig()
+
+lookup_symbols= {
+ 'A':'equatorial',
+ 'B':'arid',
+ 'C':'warm temperate',
+ 'D':'snow',
+ 'E':'polar',
+ 'W':'desert',
+ 'S':'steppe',
+ 'f':'fully humid',
+ 's':'summer dry',
+ 'w':'winter dry',
+ 'm':'monsoonal',
+ 'h':'hot arid',
+ 'k':'cold arid',
+ 'a':'hot summer',
+ 'b':'warm summer',
+ 'c':'cool summer',
+ 'd':'extremely continental',
+ 'F':'polar frost',
+ 'T':'polar tundra',
+ } 
+
+key = args.experiments.strip(' ').split(' ')[0]
+xrkoeppen = xr.open_dataset('/user/data/gent/gvo000/gvo00090/EXT/data/KOEPPEN/Koeppen-Geiger.nc')
+koeppenlookuptable = pd.DataFrame()
+koeppenlookuptable['KGCID'] = pd.Series(xrkoeppen['KGCID'])
+
+from matplotlib.patches import Rectangle,FancyBboxPatch
+
+def abline(slope, intercept,axis):
+    """Plot a line from slope and intercept"""
+    #axis = plt.gca()
+    xlim = axis.get_xlim()
+    ylim = axis.get_ylim()
+    x_vals = np.array([np.min([xlim[0],ylim[0]]),np.max([xlim[1],ylim[1]])])
+    y_vals = intercept + slope * x_vals
+    axis.plot(x_vals, y_vals, 'k--')
+
+KGCID=    ['Af', 'Am', 'As', 'Aw', 'BSh', 'BSk', 'BWh', 'BWk', 'Cfa', 'Cfb','Cfc', 'Csa', 'Csb', 'Csc', 'Cwa','Cwb', 'Cwc', 'Dfa', 'Dfb', 'Dfc','Dfd', 'Dsa', 'Dsb', 'Dsc', 'Dsd','Dwa', 'Dwb', 'Dwc', 'Dwd', 'EF','ET', 'Ocean'] 
+KGCcolors=["#960000", "#FF0000", "#FF6E6E", "#FFCCCC", "#CC8D14", "#CCAA54", "#FFCC00", "#FFFF64", "#007800", "#005000", "#003200", "#96FF00", "#00D700", "#00AA00", "#BEBE00", "#8C8C00", "#5A5A00", "#550055", "#820082", "#C800C8", "#FF6EFF", "#646464", "#8C8C8C", "#BEBEBE", "#E6E6E6", "#6E28B4", "#B464FA", "#C89BFA", "#C8C8FF", "#6496FF", "#64FFFF", "#F5FFFF"]
+
+def brightness(rrggbb):
+    """ W3C brightness definition
+        input:
+            hexadecimal color in the format:
+            #RRGGBB
+        output: value between 0 and 1
+    """
+    print(rrggbb)
+    rr = int(rrggbb[1:3],16)/int('FF',16)
+    gg = int(rrggbb[3:5],16)/int('FF',16)
+    bb = int(rrggbb[5:7],16)/int('FF',16)
+    #rr = math.floor(rrggbb/10000.)
+    #gg = math.floor((rrggbb - rr*10000.)/100.)
+    #bb = rrggbb - rr*10000 - gg*100
+    return (rr * 299. + gg * 587. + bb * 114.) / 1000.
+
+kgccolors = {}
+for iKGC,KGCname in enumerate(KGCID):
+    kgccolors[KGCname] = [KGCcolors[iKGC],'white' if (brightness(KGCcolors[iKGC])<0.5) else 'black']
+
+# kgccolors = {
+#     'Dfa':['navy','white'],
+#     'Cfb':['green','white']       ,
+#     'BSk':['tan','black']      ,
+#     'Csb':['lightgreen','black'] ,     
+#     'Cfa':['darkgreen','white']  ,    
+#     'BWh':['orange','black']      ,
+#     'Aw' :['pink','black'],
+#     'Dwc':['rebeccapurple','white'] ,    
+#     'Dfb':['darkviolet','white']    , 
+# }
+kgcnames = {
+    'Dfa':'snow \n fully humid \n hot summer',
+    'Cfb':'green'       ,
+    'BSk':'4'      ,
+    'Csb':'5'      ,
+    'Cfa':'darkgreen' ,     
+    'BWh':'6'      ,
+    'Aw' :'7'     ,
+    'Dwc':'8'     ,
+    'Dfb':'9'     ,
+    #'Dfa':'',
+}
+for KGCID in list(pd.Series(xrkoeppen['KGCID'])):
+    if KGCID not in kgcnames.keys():
+        kgcnames[KGCID] = KGCID
+    if KGCID not in kgccolors.keys():
+        kgccolors[KGCID] = ['k','k']
+
+
+koeppenlookuptable['color'] = ""
+koeppenlookuptable['textcolor'] = ""
+koeppenlookuptable['name'] = ""
+for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+    print(ikoeppen)
+    print(koeppen.KGCID)
+    print(kgccolors[koeppen.KGCID])
+    koeppenlookuptable['color'].loc[ikoeppen] = kgccolors[koeppen.KGCID][0]
+    koeppenlookuptable['textcolor'].loc[ikoeppen] = kgccolors[koeppen.KGCID][1]
+    koeppenlookuptable['name'].loc[ikoeppen] = kgcnames[koeppen.KGCID]
+
+
+
+c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname'] =  \
+    c4gldata[key].frames['stats']['records_all_stations_ini']['KGC'].map(koeppenlookuptable['KGCID'])
+
+print('sort the climate classes according to the amount ')
+koeppenlookuptable['amount'] = ""
+
+#exclude_koeppen = ['Dfc','Cwb']
+
+for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+
+    print(ikoeppen,':',koeppen)
+    kgc_select = (c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname'] == koeppen['KGCID'])
+    print(np.sum(kgc_select))
+    koeppenlookuptable.iloc[ikoeppen]['amount'] = np.sum(kgc_select)
+
+koeppenlookuptable = koeppenlookuptable.sort_values('amount',ascending=False)
+include_koeppen = list(koeppenlookuptable.KGCID)
+
+for ikey,key in enumerate(args.experiments.strip(' ').split(' ')[:2]):
+    c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname'] =  \
+        c4gldata[key].frames['stats']['records_all_stations_ini']['KGC'].map(koeppenlookuptable['KGCID'])
+
+if args.make_figures:
+    fig = plt.figure(figsize=(11,8.0))   #width,height
+    i = 1                                                                           
+    axes = {}         
+    axes_taylor = {}         
+    
+    colors = ['r','g','b','m','y','purple','orange','sienna','navy']
+    symbols = ['*','x','+']
+    dias = {}
+
+
+    i = 1
+    for varkey in ['h','theta','q']:                                                    
+        dias[varkey] =  TaylorDiagram(1., srange=[0.0,1.7],fig=fig, rect=(230+i+3),label='Reference')
+        axes[varkey] = fig.add_subplot(2,3,i)                                       
+        #axes_taylor[varkey] = fig.add_subplot(2,3,i+3)                                       
+    
+        #print(obs.std())
+        dias[varkey]._ax.axis["left"].label.set_text(\
+             "Normalized root mean square error")
+        if i == 1:
+            axes[varkey].annotate('Normalized standard deviation',\
+                        xy= (0.05,0.37),
+                        color='black',
+                        rotation=90.,
+                        xycoords='figure fraction',
+                        weight='normal',
+                        fontsize=10.,
+                        horizontalalignment='center',
+                        verticalalignment='center' ,
+                        #bbox={'edgecolor':'black',
+                        #      'boxstyle':'circle',
+                        #      'fc':koeppen.color,
+                        #      'alpha':1.0}
+                       )
+
+
+            # ticks = ticker.FuncFormatter(lambda x, pos: '{0:g}'.format(x-1.))
+            # #ax.axis["left"].axis.set_major_formatter(ticks)
+
+            # # dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+            # dias[varkey]._ax.axis["left"].axis.set_major_formatter(ticks)
+        #dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+        # Q95 = obs.quantile(0.95)
+        # Q95 = obs.quantile(0.90)
+        # Add RMS contours, and label them
+        contours = dias[varkey].add_contours(levels=5, colors='0.5') # 5 levels
+        dias[varkey].ax.clabel(contours, inline=1, fontsize=10, fmt='%.1f')
+        #dia._ax.set_title(season.capitalize())
+    
+        dias[varkey].add_grid()
+    
+    
+        #dia.ax.plot(x99,y99,color='k')
+        i += 1
+    
+
+    i = 1
+    for varkey in ['h','theta','q']:                                                    
+        #for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+        for ikey,key in enumerate(args.experiments.strip(' ').split(' ')[:2]):
+            # cc = c4gldata[key].frames['stats']['records_all_stations_ini']['cc']
+            # clearsky = (cc < 0.05)
+            # mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats'].loc[clearsky]['d'+varkey+'dt']
+            # obs = c4gldata[key].frames['stats']['records_all_stations_obs_end_obs_stats'].loc[clearsky]['d'+varkey+'dt']
+            mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt']
+            obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt']
+
+            print ('filtering classes that have sufficient samples: ', include_koeppen)
+            filter_classes = (c4gldata[key].frames['stats']['records_all_stations_ini'].KGCname.isin(include_koeppen))
+            mod = mod.loc[filter_classes]
+            obs = obs.loc[filter_classes]
+            x, y = obs.values,mod.values
+            print(key,len(obs.values))
+    
+            STD_OBS = obs.std()
+            #scores
+            PR = pearsonr(mod,obs)[0]
+            RMSE = rmse(obs,mod)                                               
+            BIAS = np.mean(mod) - np.mean(obs)
+            STD = mod.std()
+            markers=['o','^']
+            
+            # fit = np.polyfit(x,y,deg=1)
+            # axes[varkey].plot(x, fit[0] * x + fit[1],\
+            #                   color=colors[ikey],alpha=0.8,lw=2,\
+            #                   label=key+", "+\
+            #                               'R = '+str(round(PR,3))+', '+\
+            #                               'RMSE = '+str(round(RMSE,5))+units['d'+varkey+'dt']+', '+\
+            #                               'BIAS = '+str(round(BIAS,5))+units['d'+varkey+'dt'] )
+            # axes[varkey].legend(fontsize=5)
+            
+            # print(STD)
+            # print(PR)
+            dias[varkey].add_sample(STD/STD_OBS, PR,\
+                           marker=markers[ikey],ls='', mfc='white',mec='black',
+                           zorder=-100,
+                           ms=2.5*np.sqrt(np.sum(np.array(koeppenlookuptable.amount.values,dtype=np.float)))/\
+                                np.mean(np.sqrt(np.array(koeppenlookuptable.amount.values,dtype=np.float)))
+                           # annotate=koeppen.KGCID, color=koeppen.textcolor,weight='bold',fontsize=5.,\
+                           # bbox={'edgecolor':'black','boxstyle':'circle','fc':koeppen.color,'alpha':0.7}
+                           )
+            dias[varkey].add_sample(STD/STD_OBS, PR,\
+                           marker=markers[ikey],ls='', mfc='none',mec='black',
+                           zorder=700,
+                           ms=2.5*np.sqrt(np.sum(np.array(koeppenlookuptable.amount.values,dtype=np.float)))/\
+                                np.mean(np.sqrt(np.array(koeppenlookuptable.amount.values,dtype=np.float)))
+                           # annotate=koeppen.KGCID, color=koeppen.textcolor,weight='bold',fontsize=5.,\
+                           # bbox={'edgecolor':'black','boxstyle':'circle','fc':koeppen.color,'alpha':0.7}
+                           )
+            dias[varkey].add_sample(STD/STD_OBS, PR,\
+                           marker=markers[ikey],ls='', mfc='none',mec='black',
+                           zorder=700,
+                           ms=1.
+                           # annotate=koeppen.KGCID, color=koeppen.textcolor,weight='bold',fontsize=5.,\
+                           # bbox={'edgecolor':'black','boxstyle':'circle','fc':koeppen.color,'alpha':0.7}
+                           )
+            # dias[varkey].add_sample(STD/STD_OBS, PR,\
+            #                    annotate='All', alpha=1.,color='black',weight='bold',fontsize=5.,\
+            #                 zorder=100,\
+            #                         bbox={'edgecolor':'black','boxstyle':'circle','alpha':0.0}\
+            #                 )
+    
+        # put ticker position, see
+        # https://matplotlib.org/examples/ticks_and_spines/tick-locators.html 
+        # dia.ax.axis['bottom'].
+        # dia.ax.axis['left'].
+        # dia.ax.axis['left'].
+    
+        i += 1
+
+    i = 1
+    for varkey in ['h','theta','q']:                                                    
+
+        for ikey,key in enumerate(args.experiments.strip(' ').split(' ')[:2]):
+            icolor = 0
+            for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+                if koeppen.amount >= 200:
+                    print(ikoeppen,':',koeppen)
+                    kgc_select = (c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname'] == koeppen['KGCID'])
+                    koeppen_end_mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt'][kgc_select]
+                    koeppen_obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt'][kgc_select]
+    
+                    #axes[varkey].scatter(koeppen_obs,koeppen_end_mod,marker=symbols[ikoeppen],color=colors[ikey])
+                             #  label=key+", "+\
+                             #                    'R = '+str(round(PR[0],3))+', '+\
+                             #                    'RMSE = '+str(round(RMSE,5))+', '+\
+                             #                    'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+    
+    
+    
+                # # pl.scatter(obs,mod,label=key+", "+\
+                # #                              'R = '+str(round(PR[0],3))+', '+\
+                # #                              'RMSE = '+str(round(RMSE,5))+', '+\
+                # #                              'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+                    
+                    dias[varkey].add_sample(koeppen_end_mod.std()/koeppen_obs.std(),
+                                   pearsonr(koeppen_end_mod,koeppen_obs)[0],
+                                   marker=markers[ikey],linewidth=0.5,alpha=0.7,
+                                            mfc=koeppen.color,mec='black',#koeppen.color,
+                                            zorder=300+icolor,
+                                   ms=2.5*np.sqrt(koeppen.amount)/np.mean(np.sqrt(np.array(koeppenlookuptable.amount.values,dtype=np.float)))
+                                   # annotate=koeppen.KGCID, color=koeppen.textcolor,weight='bold',fontsize=5.,\
+                                   # bbox={'edgecolor':'black','boxstyle':'circle','fc':koeppen.color,'alpha':0.7}
+                                   )
+                    # dias[varkey].add_sample(koeppen_end_mod.std()/koeppen_obs.std(),
+                    #                pearsonr(koeppen_end_mod,koeppen_obs)[0],
+                    #                marker=markers[ikey],linewidth=0.5,
+                    #                         mfc=koeppen.color,mec='black',#koeppen.color,
+                    #                         zorder=301+icolor, ms=1
+                    #                # annotate=koeppen.KGCID, color=koeppen.textcolor,weight='bold',fontsize=5.,\
+                    #                # bbox={'edgecolor':'black','boxstyle':'circle','fc':koeppen.color,'alpha':0.7}
+                    #                )
+
+
+                    # dias[varkey].add_sample(koeppen_end_mod.std()/koeppen_obs.std(),
+                    #                pearsonr(koeppen_end_mod,koeppen_obs)[0],
+                    #                         marker=markers[ikey],linewidth=0.5, mfc='none',mec=str(koeppen.color),
+                    #                         zorder=600+icolor,
+                    #                ms=10.*np.sqrt(koeppen.amount)/np.mean(np.sqrt(np.array(koeppenlookuptable.amount.values,dtype=np.float)))
+                    #                # annotate=koeppen.KGCID, color=koeppen.textcolor,weight='bold',fontsize=5.,\
+                    #                # bbox={'edgecolor':'black','boxstyle':'circle','fc':koeppen.color,'alpha':0.7}
+                    #                )
+
+                    icolor += 1
+
+
+
+
+
+
+
+
+    
+            latex = {}
+            latex['dthetadt'] =  r'$d \theta / dt $'
+            latex['dqdt'] =      r'$d q / dt $'
+            latex['dhdt'] =      r'$d h / dt $'
+    
+            axes[varkey].set_xlabel('Observed')     
+
+
+            if varkey == 'q':
+                units_final = r'[$g\, kg^{-1}\, h^{-1}$]'
+            elif varkey == 'theta':
+                units_final = r'[$K\, h^{-1}$]'
+            elif varkey == 'h':
+                units_final = r'[$m\, h^{-1}$]'
+
+            axes[varkey].set_title(latex['d'+varkey+'dt']+' '+units_final,fontsize=12.)                                     
+        if i==1:                                    
+            axes[varkey].set_ylabel('Modelled')                                            
+        abline(1,0,axis=axes[varkey])
+        i +=1
+
+
+
+    axph = fig.add_axes([0,0,1,1])
+    axph.xaxis.set_visible(False)
+    axph.yaxis.set_visible(False)
+    axph.set_zorder(1000)
+    axph.patch.set_alpha(0.0)
+    
+    axph.add_patch(Rectangle((0.006,0.01), 0.99, 0.13, alpha=0.71,
+                           facecolor='white',edgecolor='grey'))
+
+
+    idx = 0
+    koeppenlookuptable_sel = koeppenlookuptable[koeppenlookuptable.amount >= 200].sort_values('KGCID',ascending=True)
+
+    for ikoepp,koepp in koeppenlookuptable_sel.iterrows():
+        xy_box = ((0.059+np.floor(idx/4)*0.335),0.127- 0.09*((np.mod(idx,4)/(len(koeppenlookuptable_sel)/4))))
+        xy_text = list(xy_box)
+        xy_text[0] += 0.009
+        xy_text[1] -= 0.00
+        xy_color = list(xy_box)
+        xy_color[0] += -0.045
+        xy_color[1] -= 0.019
+        axph.add_patch(Rectangle(xy_color,0.049,0.023,
+                                      edgecolor='black', 
+                               #       boxstyle='round', 
+                                      #xycoords='figure fraction',
+                                      fc=koepp.color,
+                                      alpha=1.0))
+    
+        axph.annotate(koepp.KGCID,
+                    xy= xy_box,
+                    color='white' if (brightness(koepp.color)<0.5) else 'black', 
+                    family='monospace',
+                    xycoords='figure fraction',
+                    weight='bold',
+                    fontsize=10.,
+                    horizontalalignment='right',
+                    verticalalignment='top' ,
+                    # bbox={'edgecolor':'black',
+                    #       'boxstyle':'round',
+                    #       'fc':koepp.color,
+                    #       'alpha':1.0}
+                   )
+        full_name = ""
+        if koepp.KGCID is not "Ocean":
+            for char in koepp.KGCID:
+                full_name += lookup_symbols[char] + ' - '
+            full_name = full_name[:-3]
+        axph.annotate(full_name,xy=xy_text,color='k'
+                    ,fontsize=8,xycoords='figure fraction',weight='bold',horizontalalignment='left',verticalalignment='top')
+
+        idx +=1
+
+    
+
+    i = 1
+    for varkey in ['h','theta','q']:                                                    
+        ikey = 0
+        key = list(args.experiments.strip().split(' '))[ikey]
+        keylabel = keylabels[ikey]
+        # cc = c4gldata[key].frames['stats']['records_all_stations_ini']['cc']
+        # clearsky = (cc < 0.05)
+    
+        # mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats'].loc[clearsky]['d'+varkey+'dt']
+        # obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats'].loc[clearsky]['d'+varkey+'dt']
+    
+        mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt']
+        obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt']
+        print ('filtering classes that have sufficient samples: ', include_koeppen)
+        filter_classes = (c4gldata[key].frames['stats']['records_all_stations_ini'].KGCname.isin(include_koeppen))
+        mod = mod.loc[filter_classes]
+        obs = obs.loc[filter_classes]
+
+    
+        nbins=40       
+        x, y = obs.values,mod.values
+        
+        xi, yi = np.mgrid[x.min():x.max():nbins*1j, y.min():y.max():nbins*1j]
+        zi = np.zeros_like(xi)*np.nan       
+        for ibin in range(nbins):
+            xmin = x.min() + ibin * (x.max() - x.min())/nbins
+            xmax = xmin + (x.max() - x.min())/nbins
+            in_bin = ((x >= xmin) & (x < xmax))
+            ybin = y[in_bin]
+            xbin = x[in_bin]
+            if len(ybin) > 20:
+                k = kde.gaussian_kde((ybin))
+                zi[ibin] = k(np.vstack([yi[ibin].flatten()]))
+        zi = zi/np.sum(zi,axis=1)[:,np.newaxis]
+        zi_int = zi.cumsum(axis=1) 
+                     #  label=key+", "+\
+                     #                    'R = '+str(round(PR[0],3))+', '+\
+                     #                    'RMSE = '+str(round(RMSE,5))+', '+\
+                     #                    'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+        axes[varkey].contour(xi, yi, zi_int.reshape(xi.shape),levels=[0.25,0.5,0.75] ,
+                colors=['darkred','lightgreen','darkred'],linewidths=[1,2,1])
+        axes[varkey].contourf(xi, yi, zi_int.reshape(xi.shape),levels=[0.25,0.75] ,
+                colors=['darkred'],alpha=0.5,)
+
+        # if varkey == 'q':
+        nani = np.concatenate([xi[zi != np.nan],yi[zi != np.nan]])
+        # axes[varkey].set_ylim((np.percentile(nani,20),np.percentile(nani,80)))
+        # #axes[varkey].set_ylim((nani.min(),nani.max()))
+        # print(varkey,(nani.min(),nani.max()))
+    
+    
+        latex = {}
+        latex['dthetadt'] =  r'$d \theta / dt $'
+        latex['dqdt'] =      r'$d q / dt $'
+        latex['dhdt'] =      r'$d h / dt $'
+    
+        axes[varkey].set_xlabel('observations')     
+
+        if varkey == 'q':
+            units_final = r'[$\mathrm{g\, kg^{-1}\, h^{-1}}$]'
+        elif varkey == 'theta':
+            units_final = r'[$\mathrm{K\, h^{-1}}$]'
+        elif varkey == 'h':
+            units_final = r'[$\mathrm{m\, h^{-1}}$]'
+
+        if varkey == 'q':
+            axes[varkey].set_title(latex['d'+varkey+'dt']+' '+units_final,fontsize=15)        
+        elif varkey == 'theta':
+            axes[varkey].set_title(latex['d'+varkey+'dt']+' '+units_final,fontsize=15)
+        elif varkey == 'h':
+            axes[varkey].set_title(latex['d'+varkey+'dt']+' '+units_final,fontsize=15)
+       #  c4gldata[key].frames['stats']['records_all_stations_ini']['KGCname']
+
+        PR = pearsonr(mod,obs)[0]
+        RMSE = rmse(obs,mod)                                               
+        BIAS = np.mean(mod) - np.mean(obs)
+        STD = mod.std()
+    
+        
+        axes[varkey].scatter(obs,mod, label='All',s=0.1,alpha=0.14,color='k')
+
+
+
+        #axes[varkey].legend(fontsize=5)
+
+        #trans = ax.get_xaxis_transform() # x in data untis, y in axes fraction
+        if varkey == 'q':
+            annotate_text = \
+                           'RMSE = '+format((RMSE*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+ '\n'+\
+                           'Bias = '+format((BIAS*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+' \n'+\
+                           r'$R$ = '+format(PR,'0.2f')
+        elif varkey == 'h':
+            annotate_text = \
+                            'RMSE = '+format(RMSE,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+                            'Bias = '+format(BIAS,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+                            r'$R$ = '+format(PR,'0.2f')
+        else:
+            annotate_text = \
+                            'RMSE = '+format(RMSE,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+                            'Bias = '+format(BIAS,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+                            r'$R$ = '+format(PR,'0.2f')
+
+
+        ann = axes[varkey].annotate(annotate_text, xy=(0.05, .98 ), xycoords='axes fraction',fontsize=9,
+       horizontalalignment='left', verticalalignment='top' 
+        )
+
+        if varkey == 'q':
+            print('get_xlim not working well...STRANGE')
+            limits =  [np.percentile(nani,1),np.percentile(nani,99)]
+        else:
+            limits =  [np.percentile(nani,1.0),np.percentile(nani,99.0)]
+
+        axes[varkey].set_xlabel('Observed')     
+        if i==1:                                    
+            axes[varkey].set_ylabel('Modelled')                                            
+        abline(1,0,axis=axes[varkey])
+
+
+        i +=1
+
+        #axes[varkey].axis('equal')
+        axes[varkey].set_aspect('equal')
+        axes[varkey].set_xlim(limits)
+        axes[varkey].set_ylim(limits)
+
+        # To specify the number of ticks on both or any single axes
+        # plt.locator_params(axis='x', nbins=6)
+        #plt.locator_params( nbins=10)
+        axes[varkey].xaxis.set_major_locator(ticker.MaxNLocator(4))
+        axes[varkey].yaxis.set_major_locator(ticker.MaxNLocator(4))
+        # axes[varkey].xaxis.set_major_locator(ticker.MultipleLocator(5))
+        # axes[varkey].yaxis.set_major_locator(ticker.MultipleLocator(5))
+
+        if varkey == 'q':
+            ticks = ticker.FuncFormatter(lambda x, pos:
+                                         '{0:g}'.format(x*1000.))
+            axes[varkey].xaxis.set_major_formatter(ticks)
+            axes[varkey].yaxis.set_major_formatter(ticks)
+
+        #     # axes[varkey].set_xticklabels(labels=ax.get_xticklabels()*1000.)
+        #     # axes[varkey].set_yticklabels(labels=ax.get_yticklabels()*1000.)
+    
+    
+    # # legend for different forcing simulations (colors)
+    # ax = fig.add_axes([0.05,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    # leg = []
+    # for ikey,key in enumerate(args.experiments.strip().split(' ')):
+    #     leg1, = ax.plot([],colors[ikey]+markers[ikey] ,markersize=10)
+    #     leg.append(leg1)
+    # ax.axis('off')
+    # #leg1 =
+    # ax.legend(leg,list(args.experiments.strip().split(' ')),loc=2,fontsize=10)
+    
+    
+    # # legend for different stations (symbols)
+    # ax = fig.add_axes([0.25,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    # leg = []
+    # isymbol = 0
+    # for icurrent_station,current_station in c4gldata[key].frames['worldmap']['stations'].table.iterrows():
+    #     leg1, = ax.plot([],'k'+symbols[isymbol] ,markersize=10)
+    #     leg.append(leg1)
+    #     isymbol += 1
+    # 
+    # # symbol for all stations
+    # leg1, = ax.plot([],'ko',markersize=10)
+    # leg.append(leg1)
+    
+    # ax.axis('off')
+    # ax.legend(leg,['HUMPPA','BLLAST','GOAMAZON','All'],loc=2,fontsize=10)
+    
+    
+    fig.subplots_adjust(top=0.95,bottom=0.20,left=0.08,right=0.94,hspace=0.35,wspace=0.29)
+    
+    
+    #pl.legend(leglist,('EMI:WOC','EMI:MED','EMI:BEC'),loc=2,fontsize=16,prop={'family':
+    # figfn = '/user/data/gent/gvo000/gvo00090/D2D/archive/report/global_eval_report_cs.png'
+    # fig.savefig(figfn,dpi=200); print("Image file written to:", figfn)
+    
+    if args.figure_filename is not None:
+        fig.savefig(args.figure_filename,dpi=200); print("Image file written to:",args.figure_filename)
+        fig.savefig(args.figure_filename.replace('png','pdf')); print("Image file written to:", args.figure_filename)
+    fig.show()  
+
+    koeppenlookuptable = koeppenlookuptable.sort_index()
+    if args.show_control_parameters == 'True':
+
+
+        pkmn_type_colors = [
+                                            '#A0A0A0',  # Poison
+                                            '#78C850',  # Grass
+                                            '#F08030',  # Fire
+                                            '#6890F0',  # Water
+                                            '#F08030',  # Fire
+                                            '#C03028',  # Fighting
+                                            '#F85888',  # Psychic
+                                            '#A8B820',  # Bug
+                                            '#A8A878',  # Normal
+                                            '#F8D030',  # Electric
+                                            '#E0C068',  # Ground
+                                            '#EE99AC',  # Fairy
+                                            '#B8A038',  # Rock
+                                            '#705898',  # Ghost
+                                            '#98D8D8',  # Ice
+                                            '#7038F8',  # Dragon
+                                           ]
+
+
+
+        #sns.set()
+        #fig = pl.figure(figsize=(11,7))
+        i = 1
+        #axes = {}
+        data_all = pd.DataFrame()
+        data_input = pd.DataFrame()
+        
+        
+        
+        # #for varkey in ['theta','q']:     
+        # EF =\
+        #     c4gldata[key].frames['stats']['records_all_stations_ini'].BR/(1.+\
+        #     c4gldata[key].frames['stats']['records_all_stations_ini'].BR)
+        # EF[EF<0] = np.nan
+        # EF[EF>1] = np.nan
+        
+        # c4gldata[key].frames['stats']['records_all_stations_ini']['EF'] = EF
+        
+        ikey = 0
+        key = list(args.experiments.strip().split(' '))[ikey]
+        data_all = pd.DataFrame()
+
+        tempdatamodstats = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_end_obs_stats'].copy())
+        
+
+        tempdatamodstats["source"] = "soundings"
+        tempdatamodstats["source_index"] = "soundings"
+
+        ini_ref = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_ini'].copy())
+        tempdataini_this = pd.DataFrame(ini_ref.copy())
+
+        tempdatamodstats['dates']= tempdataini_this.ldatetime.dt.date
+        tempdatamodstats['STNID']= tempdataini_this.STNID
+        tempdatamodstats['source']= "soundings"
+        tempdatamodstats['source_index']= "soundings"
+        tempdatamodstats.set_index(['source_index','STNID','dates'],inplace=True)
+        #print('hello')
+
+        tempdataini = pd.DataFrame(ini_ref)
+        tempdataini["source"] = "soundings"
+        tempdataini["source_index"] = "soundings"
+        tempdataini = tempdataini.set_index(['source_index','STNID','dates'])
+        #print('hello2')
+
+
+        data_all = pd.concat([data_all,tempdatamodstats],axis=0)
+        data_input = pd.concat([data_input,tempdataini],axis=0)
+        #print(data_input.shape)
+        #print(data_all.shape)
+
+            
+        for ikey,key in enumerate(list(args.experiments.strip().split(' '))):
+            keylabel = keylabels[ikey]
+
+            tempdatamodstats = pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_end_mod_stats'].copy())
+            tempdataini_this= pd.DataFrame(c4gldata[key].frames['stats']['records_all_stations_ini'].copy())
+
+            tempdatamodstats['dates']= tempdataini_this.ldatetime.dt.date
+            tempdatamodstats['STNID']= tempdataini_this.STNID
+            tempdatamodstats['source']= keylabel
+            tempdatamodstats['source_index']= keylabel
+            tempdatamodstats.set_index(['source_index','STNID','dates'],inplace=True)
+            #print('hello')
+
+
+            tempdataini = pd.DataFrame(ini_ref.copy())
+            tempdataini["source"] = keylabel
+            tempdataini["source_index"] = keylabel
+            tempdataini = tempdataini.set_index(['source_index','STNID','dates'])
+    
+
+            index_intersect = tempdataini.index.intersection(tempdatamodstats.index)
+
+            tempdataini = tempdataini.loc[index_intersect]
+            #print('hello4')
+            tempdatamodstats = tempdatamodstats.loc[index_intersect]
+            #print('hello5
+
+            # data[varkey] = tempdatamodstats['d'+varkey+'dt']
+            data_all = pd.concat([data_all,tempdatamodstats],axis=0)
+            data_input = pd.concat([data_input, tempdataini],axis=0)
+            #print(data_input.shape)
+            #print(data_all.shape)
+
+        data_input.cc = data_input.cc.clip(0.,+np.inf)
+
+        for varkey in ['h','theta','q']:
+            varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+            data_all = data_all.rename(columns={'d'+varkey+'dt':varkey_full})
+            data_all['KGCname'] = data_input['KGCname']
+
+
+
+
+            #print(data_input.shape)
+            #print(data_all.shape)
+        # xrkoeppen = xr.open_dataset('/user/data/gent/gvo000/gvo00090/EXT/data/KOEPPEN/Koeppen-Geiger.nc')
+        # lookuptable = pd.Series(xrkoeppen['KGCID'])
+        # data_all['KGCname'] = data_input['KGC'].map(lookuptable)
+        #print('hello6')
+        #print(data_all.columns)
+        #print('hello7')
+
+        
+
+
+        varkeys = ['h','theta','q']
+        for varkey in varkeys:
+            #input_keys =['wg','cc']
+            #for input_key in input_keys:
+            varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+
+            #print('hello8')
+            #print(data_input.shape)
+            #print(data_all.shape)
+            #input_key_full = input_key + "["+units[input_key]+"]"
+            #print('hello9')
+            #print(data_input.shape)
+            #print(data_all.shape)
+            print ('Excluding extreme values from the classes plots')
+            qvalmax = data_all[varkey_full].quantile(0.999)
+            qvalmin = data_all[varkey_full].quantile(0.001)
+            select_data = (data_all[varkey_full] >= qvalmin) & (data_all[varkey_full] < qvalmax)
+            #print('hello11')
+            data_all = data_all[select_data]
+            #print('hello12')
+            data_input = data_input[select_data.values]
+
+            data_input = data_input[data_all.KGCname.isin(list(koeppenlookuptable.KGCID))]
+            data_all = data_all[data_all.KGCname.isin(list(koeppenlookuptable.KGCID))]
+            #print('hello13')
+            #print(data_input.shape)
+            #print(data_all.shape)
+            #print('hello10')
+            
+
+        #sns.set(style="ticks", palette="deep")
+
+        
+        exppairs = {'obs|ref'     :['soundings',keylabels[0]],
+                    'fcap|wilt'   :[keylabels[1] ,keylabels[2]],
+                    'allveg|noveg':[keylabels[3],keylabels[4]]
+                   }
+        current_palette = sns.color_palette('deep')
+        exppalettes = {'obs|ref'     :['white','grey'],
+                    'fcap|wilt'   :[current_palette[0],current_palette[3]],
+                       'allveg|noveg':[current_palette[2],current_palette[8]]
+                   }
+
+        data_all['expname'] = ""
+        print('making alternative names for legends')
+        expnames = {'soundings':'obs',\
+                    keylabels[0]:'ref',\
+                    keylabels[1]:'dry',\
+                    keylabels[2]:'wet',\
+                    keylabels[3]:'noveg',\
+                    keylabels[4]:'fullveg',\
+                   }
+        for expname_orig,expname in expnames.items():
+            data_all['expname'][data_all['source'] == expname_orig] = expname
+
+        data_all['exppair'] = ""
+        for exppairname,exppair in exppairs.items():
+            data_all['exppair'][  data_all['source'].isin(exppair)  ] = exppairname
+
+        icolor = 0
+        
+        koeppenlookuptable = koeppenlookuptable[koeppenlookuptable.amount >= 200]
+        koeppenlookuptable = koeppenlookuptable.sort_values('amount',ascending=False)
+        koeppenlookuptable = koeppenlookuptable[:9]
+        fig, axes = plt.subplots(nrows=len(varkeys)*len(koeppenlookuptable), \
+                                 ncols=len(exppairs), \
+                                 figsize=(8, 13), #width, height
+                                 sharex='col',
+                                 #gridspec_kw=dict(height_ratios=(1, 3), 
+                                 gridspec_kw=dict(hspace=0.20,wspace=0.08,top=0.94,bottom=0.06,left=0.15,right=0.99))
+
+        varkey = 'q'
+        data_all['d'+varkey+'dt ['+units[varkey]+'/h]'] *= 1000.  
+
+        icol = 0
+        irow = 0
+        sns.set_style('whitegrid')
+        for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+                for exppairname,exppair in exppairs.items():
+                    for varkey in varkeys:
+                        varkey_full = 'd'+varkey+'dt ['+units[varkey]+'/h]'
+                        ax = axes[irow,icol]
+
+                     #   axes[i] = fig.add_subplot(len(varkeys)*len(koeppenlookuptable),len(exppairs),icolor)
+                #sns.violinplot(x='KGC',y=varkey_full,data=data_all,hue='source',linewidth=2.,palette="muted",split=True,inner='quart') #,label=key+", R = "+str(round(PR[0],3)),data=data)       
+                
+                #ax.set_title(input_key_full)
+                        current_data = data_all[(data_all['exppair'] == exppairname) & (data_all['KGCname']  == koeppen.KGCID)]
+                        sns.violinplot(y='exppair', x=varkey_full,
+                                            hue="expname",split=True,
+                                 palette=exppalettes[exppairname],
+                                # palette=["m", "g",'r','b'],
+                                 linewidth=1.0,inner='quart',
+                                            data=current_data,sym='',legend=False,ax=ax)
+                        ax.legend("")
+                        ax.legend_.draw_frame(False)
+                        ax.set_yticks([])
+                        ax.set_ylabel("")
+
+                        # if varkey == 'q':
+                        #     ticks = ticker.FuncFormatter(lambda x, pos:
+                        #                                  '{0:g}'.format(x*1000.))
+                        #     ax.xaxis.set_major_formatter(ticks)
+
+                        if varkey == 'q':
+                            title_final = r'$dq/dt$'
+                            xlabel_final = r'[$\mathrm{g\, kg^{-1}\, h^{-1}}$]'
+                        elif varkey == 'theta':
+                            title_final = r'$d\theta/dt$'
+                            xlabel_final = r'[$\mathrm{K\, h^{-1}}$]'
+                        elif varkey == 'h':
+                            title_final = r'$dh/dt$'
+                            xlabel_final = r'[$\mathrm{m\, h^{-1}}$]'
+
+
+                        ax.set_xlabel("")
+                        #sns.despine(left=True, right=True, bottom=False, top=False)
+                        if irow == (len(varkeys)*len(koeppenlookuptable)-1):
+                            #ax.set_frame_on(False)
+
+                            ax.set_xlabel(xlabel_final)
+                            ax.tick_params(top='off', bottom='on', left='off',
+                                            right='off', labelleft='off',
+                                            labeltop='off',
+                                            labelbottom='on'
+                                          )
+                            ax.spines['top'].set_visible(False)
+                            ax.spines['bottom'].set_visible(True)
+                            ax.spines['left'].set_visible(True)
+                            ax.spines['right'].set_visible(True)
+                            #sns.despine(left=True, right=True, bottom=True, top=False)
+                        elif irow == 0:
+                            ax.set_title(title_final,fontsize=17.)
+                            ax.tick_params(top='off', bottom='off', left='off',
+                                            right='off', labelleft='off',
+                                            labelbottom='off')
+                            #ax.set_frame_on(False)
+                            # ax.spines['left'].set_visible(True)
+                            # ax.spines['right'].set_visible(True)
+                            ax.spines['top'].set_visible(True)
+                            ax.spines['bottom'].set_visible(False)
+                            ax.spines['left'].set_visible(True)
+                            ax.spines['right'].set_visible(True)
+                            #sns.despine(left=True, right=True, bottom=False, top=True)
+                            #ax.axis("off")
+                        elif np.mod(irow,len(exppairs)) == 0:
+                            ax.tick_params(top='off', bottom='off', left='off',
+                                            right='off', labelleft='off',
+                                            labelbottom='off')
+                            #ax.set_frame_on(False)
+                            # ax.spines['left'].set_visible(True)
+                            # ax.spines['right'].set_visible(True)
+                            ax.spines['top'].set_visible(True)
+                            ax.spines['bottom'].set_visible(False)
+                            ax.spines['left'].set_visible(True)
+                            ax.spines['right'].set_visible(True)
+                            #sns.despine(left=True, right=True, bottom=False, top=True)
+                            #ax.axis("off")
+                        elif np.mod(irow,len(exppairs)) == 2:
+                            ax.tick_params(top='off', bottom='on', left='off',
+                                            right='off', labelleft='off',
+                                            labelbottom='off')
+                            #ax.set_frame_on(False)
+                            # ax.spines['left'].set_visible(True)
+                            # ax.spines['right'].set_visible(True)
+                            ax.spines['top'].set_visible(False)
+                            ax.spines['bottom'].set_visible(True)
+                            ax.spines['left'].set_visible(True)
+                            ax.spines['right'].set_visible(True)
+                            #sns.despine(left=True, right=True, bottom=False, top=True)
+                            #ax.axis("off")
+                        else:
+                            ax.tick_params(top='off', bottom='off', left='off',
+                                            right='off', labelleft='off',
+                                            labelbottom='off')
+                            #ax.set_frame_on(False)
+                            #ax.spines['left'].set_visible(True)
+                            #ax.spines['right'].set_visible(True)
+                            ax.spines['top'].set_visible(False)
+                            ax.spines['bottom'].set_visible(False)
+                            ax.spines['left'].set_visible(True)
+                            ax.spines['right'].set_visible(True)
+                            #ax.axis("off")
+                        icol +=1
+                    irow +=1
+                    icol=0
+
+        idx = 0
+        for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+            ax.annotate(koeppen.KGCID,
+                        xy= (0.01,0.09 + (1.-0.12)*(1. - (idx+.5)/len(koeppenlookuptable))),
+                        color=koeppen.textcolor, 
+                        family='monospace',
+                        xycoords='figure fraction',
+                        weight='bold',
+                        fontsize=8.,
+                        horizontalalignment='top',
+                        verticalalignment='left' ,
+                        bbox={'edgecolor':'black',
+                              'boxstyle':'circle',
+                              'fc':koeppen.color,
+                              'alpha':1.0}
+                       )
+            data_select = data_input[(data_input['source'] == 'soundings') & (data_input['KGCname']  == koeppen.KGCID)]
+            ax.annotate('#:   '+r'$'+str(koeppen.amount)+' '+'$'+'\n'+\
+                        'sm: '+r'$'+str(round(data_select.wg.mean(),2))+'$'+r'$\, \mathrm{m^{3}\, m^{-3}}$'+'\n'+\
+                        'vf: '+str(round(data_select.cveg.mean(),2))+'\n'+\
+                        'cc:  '+str(round(data_select.cc.mean(),2))+'\n'+\
+                        'lat: '+r'$'+str(int(data_select.latitude.mean()))+r'\, ^\circ$'+' \n',\
+                        xy= (0.01,0.015 + (1.-0.12)*(1. - (idx+.5)/len(koeppenlookuptable))),
+                        color='black',
+                        family='monospace',
+                        xycoords='figure fraction',
+                        weight='normal',
+                        fontsize=8.,
+                        horizontalalignment='top',
+                        verticalalignment='left' ,
+                        #bbox={'edgecolor':'black',
+                        #      'boxstyle':'circle',
+                        #      'fc':koeppen.color,
+                        #      'alpha':1.0}
+                       )
+            idx+=1
+
+
+
+
+            # if i ==1:
+            #      plt.legend(loc='upper right',fontsize=7.,frameon=True,framealpha=0.7)
+            # else:
+            #      ax.get_legend().set_visible(False)
+            # #     plt.legend('off')
+            # if i >= 3:
+            #     idx = 0
+            #     for ikoeppen,koeppen in koeppenlookuptable.iterrows():
+
+            #         ax.annotate(koeppen.KGCID,
+            #                     xy=((idx+.5)/len(koeppenlookuptable),-0.00),
+            #                     color=koeppen.textcolor, 
+            #                     xycoords='axes fraction',
+            #                     weight='bold',
+            #                     fontsize=8.,
+            #                     horizontalalignment='center',
+            #                     verticalalignment='center' ,
+            #                     bbox={'edgecolor':'black',
+            #                           'boxstyle':'circle',
+            #                           'fc':koeppen.color,
+            #                           'alpha':1.0}
+            #                    )
+            #         idx+=1
+            #     ax.set_xticklabels([])#labels=ax.get_xticklabels())
+            #     ax.set_xlabel('Köppen climate class')
+            # else:
+            #     ax.set_xticklabels([])
+            #     ax.set_xlabel('')
+
+            # # ax.set_yticklabels([])
+            # # ax.set_ylabel('')
+            # if varkey == 'q':
+            #     ticks = ticker.FuncFormatter(lambda x, pos:
+            #                                  '{0:g}'.format(x*1000.))
+            #     #ax.xaxis.set_major_formatter(ticks)
+            #     ax.yaxis.set_major_formatter(ticks)
+
+            #     ax.set_ylabel(latex['d'+varkey+'dt']+' ['+r'$10^{-3} \times $'+units['d'+varkey+'dt']+']')        
+            # else:
+            #     ax.set_ylabel(latex['d'+varkey+'dt']+' ['+units['d'+varkey+'dt']+']')        
+
+
+
+
+            # for j,artist in enumerate(ax.artists):
+            #     if np.mod(j,len(list(args.experiments.strip().split(' ')))+1) !=0:
+            #         # Set the linecolor on the artist to the facecolor, and set the facecolor to None
+            #         #print(j,artist)
+            #         col = artist.get_facecolor()
+            #         #print(j,artist)
+            #         artist.set_edgecolor(col)
+            #         #print(j,artist)
+            #         artist.set_facecolor('None')
+            # 
+            #         # Each box has 6 associated Line2D objects (to make the whiskers, fliers, etc.)
+            #         # Loop over them here, and use the same colour as above
+            #         
+            #         for k in range(j*5,j*5+5):
+            #             line = ax.lines[k]
+            #             line.set_color(col)
+            #             line.set_mfc(col)
+            #             line.set_mec(col)
+            # 
+            # # Also fix the legend
+            # j = 0
+            # for legpatch in ax.get_legend().get_patches():
+            #     if j > 0:
+
+            #         col = legpatch.get_facecolor()
+            #         legpatch.set_edgecolor(col)
+            #         legpatch.set_facecolor('None')
+            #     j +=1
+
+
+
+
+
+            #ax.grid()
+            #sns.despine(offset=10, trim=True)
+        fig.tight_layout()
+        # fig.subplots_adjust(
+        #     bottom=0.12,left=0.15,top=0.99,right=0.99,wspace=0.10,hspace=0.05,)
+        if args.figure_filename_2 is not None:
+            fig.savefig(args.figure_filename_2,dpi=200); print("Image file written to:", args.figure_filename_2)
+
+            fig.savefig(args.figure_filename_2.replace('png','pdf')); print("Image file written to:", args.figure_filename_2)
+        fig.show()
+
+
+
diff --git a/class4gl/interface/interface_show_profiles.py b/class4gl/interface/interface_show_profiles.py
new file mode 100644
index 0000000..c15c5ea
--- /dev/null
+++ b/class4gl/interface/interface_show_profiles.py
@@ -0,0 +1,567 @@
+import numpy as np
+
+import pandas as pd
+import sys
+
+import matplotlib
+matplotlib.use('TkAgg')
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--experiments')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--load_globaldata',default=False) # load the data needed for the interface
+parser.add_argument('--make_figures',default=None)
+parser.add_argument('--figure_filename',default=None)
+parser.add_argument('--tendencies_revised',default=False)
+parser.add_argument('--obs_filter',default="True")
+args = parser.parse_args()
+
+print('Adding python library:',args.c4gl_path_lib)
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import c4gl_interface_soundings,get_record_yaml
+from class4gl import class4gl_input, data_global,class4gl,units
+#from sklearn.metrics import mean_squared_error
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+#import seaborn.apionly as sns
+import pylab as pl
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import kde
+from scipy.stats import pearsonr                                                
+from taylorDiagram import TaylorDiagram
+from matplotlib import ticker
+# import importlib
+# importlib.reload(mpl); importlib.reload(plt); importlib.reload(sns)
+
+
+
+
+
+latex = {}
+latex['dthetadt'] =  r'$d \theta / dt $'
+latex['dqdt'] =      r'$d q / dt $'
+latex['dhdt'] =      r'$d h / dt $'
+
+def abline(slope, intercept,axis):
+    """Plot a line from slope and intercept"""
+    #axis = plt.gca()
+    x_vals = np.array(axis.get_xlim())
+    y_vals = intercept + slope * x_vals
+    axis.plot(x_vals, y_vals, 'k--')
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+
+
+
+# EXPS  =\
+# {
+# 'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_ADV':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'IOPS_ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+# }
+
+if args.load_globaldata:
+    # iniitialize global data
+    globaldata = data_global()
+    # ...  and load initial data pages
+    globaldata.load_datasets(recalc=0)
+else:
+    globaldata = None
+
+c4gldata = []
+    
+c4gldata.append(c4gl_interface_soundings( \
+                  '/data/gent/vo/000/gvo00090/D2D/data/C4GL/20181030/IOPS_ADV_ITER/',\
+                  '/data/gent/vo/000/gvo00090/D2D/data/SOUNDINGS/IOPS/',\
+                  globaldata,\
+                  refetch_records=False,\
+                  obs_filter = True,\
+                  tendencies_revised = args.tendencies_revised\
+                ))
+c4gldata.append(c4gl_interface_soundings( \
+                  '/data/gent/vo/000/gvo00090/D2D/data/C4GL/20181017_NOON/IOPS_ADV_ITER/',\
+                  '/data/gent/vo/000/gvo00090/D2D/data/SOUNDINGS/IOPS_NOON/',\
+                  globaldata,\
+                  refetch_records=False,
+                  obs_filter = False,
+                  tendencies_revised = args.tendencies_revised
+                ))
+
+
+profiles_morning            = []
+profiles_morning_raw        = []
+profiles_noon_obs           = []
+profiles_noon_obs_raw       = []
+profiles_noon_mod           = []
+profiles_afternoon_obs      = []
+profiles_afternoon_obs_raw  = []
+profiles_afternoon_mod      = []
+profiles_location      = []
+profiles_morning_datetime      = []
+profiles_noon_datetime      = []
+profiles_afternoon_datetime      = []
+
+for i in range(15):
+
+    if i == 0:
+        location = 'HUMPPA'
+        c4gldata[0].sel_station(90000)
+        c4gldata[1].sel_station(90000)
+    elif i == 7:
+        location = 'BLLAST'
+        c4gldata[0].sel_station(90001)
+        c4gldata[1].sel_station(90001)
+    elif i == 11:
+        location = 'GOAMAZON'
+        c4gldata[0].sel_station(90002)
+        c4gldata[1].sel_station(90002)
+
+    # if i == 12:
+    #     c4gldata[1].next_record()
+    #     c4gldata[1].next_record()
+
+        
+    profiles_location.append(str(location))
+    profiles_morning_datetime.append(c4gldata[0].frames['profiles']['record_yaml_ini'].pars.ldatetime)
+    profiles_morning.append( c4gldata[0].frames['profiles']['record_yaml_ini'].air_ap)
+    profiles_morning_raw.append( c4gldata[0].frames['profiles']['record_yaml_ini'].air_balloon)
+
+    profiles_noon_datetime.append(c4gldata[1].frames['profiles']['record_yaml_obs_afternoon'].pars.ldatetime)
+
+    if i in [12,13]:
+        profiles_noon_datetime.append(c4gldata[1].frames['profiles']['record_yaml_obs_afternoon'].pars.ldatetime)
+        profiles_noon_obs.append(c4gldata[1].frames['profiles']['record_yaml_obs_afternoon'].air_ap)
+        profiles_noon_obs_raw.append(c4gldata[1].frames['profiles']['record_yaml_obs_afternoon'].air_balloon)
+        profiles_noon_mod.append(c4gldata[1].frames['profiles']['record_yaml_mod'].air_ap)
+        profiles_noon_obs[-1].theta = np.nan
+        profiles_noon_obs_raw[-1].theta = np.nan
+        profiles_noon_mod[-1].theta = np.nan
+        profiles_noon_obs[-1].q = np.nan
+        profiles_noon_obs_raw[-1].q = np.nan
+        profiles_noon_mod[-1].q = np.nan
+
+    else:
+        profiles_noon_obs.append( c4gldata[1].frames['profiles']['record_yaml_obs_afternoon'].air_ap)
+        profiles_noon_obs_raw.append( c4gldata[1].frames['profiles']['record_yaml_obs_afternoon'].air_balloon)
+        profiles_noon_mod.append( c4gldata[1].frames['profiles']['record_yaml_mod'].air_ap)
+
+    profiles_afternoon_datetime.append(c4gldata[0].frames['profiles']['record_yaml_obs_afternoon'].pars.ldatetime)
+    profiles_afternoon_obs.append( c4gldata[0].frames['profiles']['record_yaml_obs_afternoon'].air_ap)
+    profiles_afternoon_obs_raw.append(c4gldata[0].frames['profiles']['record_yaml_obs_afternoon'].air_balloon)
+    profiles_afternoon_mod.append( c4gldata[0].frames['profiles']['record_yaml_mod'].air_ap)
+
+    if i < 14:
+        c4gldata[0].next_record()
+        if i not in [9,10]:
+            c4gldata[1].next_record()
+
+
+
+
+            #axes[varkey].set_title(latex['d'+varkey+'dt']+' '+units_final,fontsize=12.)                                     
+
+
+
+
+
+numprofiles = len(profiles_afternoon_mod)
+
+
+per_row = 4.
+VARS = ['theta','q']
+
+for var in VARS:
+    fig = plt.figure(figsize=(9,8.5))
+    for i in range(numprofiles):
+        ax = fig.add_subplot(np.ceil(numprofiles/per_row),per_row,i+1)
+    
+        if var == 'q':
+            fac = 1000.
+        else:
+            fac = 1.
+    
+        label = profiles_morning_datetime[i].strftime("%H:%m")+'ST'+' ini'
+        plt.plot(profiles_morning[i][var].values*fac,profiles_morning[i].z.values,'b--',label=label)
+        #plt.plot(profiles_morning_raw[i][var].values,profiles_morning_raw[i].z.values,'b*')
+        
+        label = profiles_noon_datetime[i].strftime("%H:%m")+'ST'+' obs'
+        #label = profiles_morning_datetime[i].hour()+':'+profiles_morning_datetime[i].minute()+'ST')+' ini')
+        plt.plot(profiles_noon_obs[i][var].values*fac,profiles_noon_obs[i].z.values,'g--',label=label)
+        #plt.plot(profiles_noon_obs_raw[i][var].values,profiles_noon_obs_raw[i].z.values,'g*')
+        label = profiles_noon_datetime[i].strftime("%H:%m")+'ST'+' mod'
+        plt.plot(profiles_noon_mod[i][var].values*fac,profiles_noon_mod[i].z.values,'g-',label=label)
+        
+        
+        label = profiles_afternoon_datetime[i].strftime("%H:%m")+'ST'+' obs'
+        plt.plot(profiles_afternoon_obs[i][var].values*fac, profiles_afternoon_obs[i].z.values,'r--',label=label)
+        #plt.plot(profiles_afternoon_obs_raw[i][var].values,profiles_afternoon_obs_raw[i].z.values,'r*')
+        label = profiles_afternoon_datetime[i].strftime("%H:%m")+'ST'+' mod'
+        plt.plot(profiles_afternoon_mod[i][var].values*fac, profiles_afternoon_mod[i].z.values,'r-',label=label)
+        if var == 'q':
+            if i >=(numprofiles - per_row):
+                ax.legend(loc=2,fontsize=6)
+            else:
+                ax.legend(loc=1,fontsize=6)
+        else:
+            ax.legend(loc=2,fontsize=6)
+    
+    
+        ax.set_ylim(0,3500)
+        if var == 'theta':
+             ax.set_xlim((283.,315.))
+        elif var == 'q':
+             ax.set_xlim((0.,21.))
+        ax.set_title(profiles_location[i]+' '+ str(profiles_morning_datetime[i].date()))
+    
+        if (np.mod(i,per_row) !=0):
+            ax.set_yticklabels([])
+        else:
+            ax.set_ylabel('$h$ [$\mathrm{m}$]')
+    
+        if i < (numprofiles - per_row):
+            ax.set_xticklabels([])
+        else:
+           if var == 'q':
+               units_final = r'$q$ [$\mathrm{g\, kg^{-1}}$]'
+           elif var == 'theta':
+               units_final = r'$\theta$ [$\mathrm{K}$]'
+           ax.set_xlabel(units_final)
+
+
+
+
+    #ax.tick_params(
+    #axis='x',          # changes apply to the x-axis
+    #which='both',      # both major and minor ticks are affected
+    #left=(np.mod(i,per_row) ==0),      # ticks along the bottom edge are off
+    #top=False,         # ticks along the top edge are off
+    #labelbottom= (i==np.ceil(numprofiles/per_row)) # labels along the bottom edge are off
+    #)
+
+    fig.tight_layout()
+    fig.subplots_adjust(wspace=0.05,right=0.99,hspace=0.25)
+    fig.show()
+
+
+
+# if bool(args.make_figures):
+#     fig = plt.figure(figsize=(10,7))   #width,height
+#     i = 1                                                                           
+#     axes = {}         
+#     axes_taylor = {}         
+#     
+#     #colors = ['r','g','b','m']
+#     colors = ['k']
+#     symbols = ['^','x','+']
+#     dias = {}
+#     
+#     for varkey in ['h','theta','q']:                                                    
+#         axes[varkey] = fig.add_subplot(2,3,i)                                       
+#         #axes_taylor[varkey] = fig.add_subplot(2,3,i+3)                                       
+#     
+#         #print(obs.std())
+#         obs = c4gldata[args.experiments.strip().split()[0]].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt']
+#         STD_OBS = obs.std()
+#         dias[varkey] =  TaylorDiagram(1., srange=[0.0,1.7],fig=fig, rect=(230+i+3),label='Reference')
+#         dias[varkey]._ax.axis["left"].label.set_text(\
+#             "Normalized standard deviation")
+#         if i == 1:
+#             axes[varkey].annotate('Normalized standard deviation',\
+#                         xy= (0.05,0.36),
+#                         color='black',
+#                         rotation=90.,
+#                         xycoords='figure fraction',
+#                         weight='normal',
+#                         fontsize=10.,
+#                         horizontalalignment='center',
+#                         verticalalignment='center' ,
+#                         #bbox={'edgecolor':'black',
+#                         #      'boxstyle':'circle',
+#                         #      'fc':koeppen.color,
+#                         #      'alpha':1.0}
+#                        )
+#         # dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+#         # dias[varkey]._ax.axis["left"].axis.set_major_locator(np.arange(0.,2.,0.25))
+#         #dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+#         # Q95 = obs.quantile(0.95)
+#         # Q95 = obs.quantile(0.90)
+#         # Add RMS contours, and label them
+#         contours = dias[varkey].add_contours(levels=5, colors='0.5') # 5 levels
+#         dias[varkey].ax.clabel(contours, inline=1, fontsize=10, fmt='%.1f')
+#         #dia._ax.set_title(season.capitalize())
+#     
+#         dias[varkey].add_grid()
+#     
+#     
+#         #dia.ax.plot(x99,y99,color='k')
+#     
+#         
+#         for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+#             mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats']['d'+varkey+'dt']
+#             obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt']
+#             x, y = obs.values,mod.values
+#             print(key,len(obs.values))
+#     
+#             #scores
+#             PR = pearsonr(mod,obs)[0]
+#             RMSE = rmse(obs,mod)                                               
+#             BIAS = np.mean(mod) - np.mean(obs)
+#             STD = mod.std()
+#             
+#             fit = np.polyfit(x,y,deg=1)
+# 
+#             if varkey == 'q':
+#                 axes[varkey].plot(x, fit[0] * x + fit[1],\
+#                                   color=colors[ikey],alpha=0.8,lw=2,\
+#                                   label=key+", "+\
+#                            'RMSE = '+format((RMSE*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+ '\n'+\
+#                            'Bias = '+format((BIAS*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+' \n'+\
+#                            r'$R$ = '+format(PR,'0.2f') )
+# 
+# 
+#             elif varkey == 'h':
+#                 axes[varkey].plot(x, fit[0] * x + fit[1],\
+#                                   color=colors[ikey],alpha=0.8,lw=2,\
+#                                   label=key+", "+\
+#                             'RMSE = '+format(RMSE,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+#                             'Bias = '+format(BIAS,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+#                             r'$R$ = '+format(PR,'0.2f'))
+#             else: #theta
+#                 axes[varkey].plot(x, fit[0] * x + fit[1],\
+#                                   color=colors[ikey],alpha=0.8,lw=2,\
+#                                   label=key+", "+\
+#                             'RMSE = '+format(RMSE,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+#                             'Bias = '+format(BIAS,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+#                             r'$R$ = '+format(PR,'0.2f'))
+# 
+#             if varkey == 'q':
+#                 annotate_text = \
+#                                'RMSE = '+format((RMSE*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+ '\n'+\
+#                                'Bias = '+format((BIAS*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+' \n'+\
+#                                r'$R$ = '+format(PR,'0.2f')
+#                 ann = axes[varkey].annotate(annotate_text, xy=(0.95, .05 ), xycoords='axes fraction',fontsize=9,
+#        horizontalalignment='right', verticalalignment='bottom' ,
+#         bbox={'edgecolor':'black',
+#                           'fc':'white',  
+#                               'boxstyle':'square',
+#                               'alpha':0.8}
+#                                        )
+#             elif varkey == 'h':
+#                 annotate_text = \
+#                                 'RMSE = '+format(RMSE,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+#                                 'Bias = '+format(BIAS,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+#                                 r'$R$ = '+format(PR,'0.2f')
+#                 ann = axes[varkey].annotate(annotate_text, xy=(0.05, .97 ), xycoords='axes fraction',fontsize=9,
+#        horizontalalignment='left', verticalalignment='top' ,
+#         bbox={'edgecolor':'black',
+#                           'fc':'white',  
+#                               'boxstyle':'square',
+#                               'alpha':0.8}
+#                                        )
+#             else:
+#                 annotate_text = \
+#                                 'RMSE = '+format(RMSE,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+#                                 'Bias = '+format(BIAS,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+#                                 r'$R$ = '+format(PR,'0.2f')
+# 
+#                 ann = axes[varkey].annotate(annotate_text, xy=(0.05, .97 ), xycoords='axes fraction',fontsize=9,
+#        horizontalalignment='left', verticalalignment='top' ,
+#         bbox={'edgecolor':'black',
+#                           'fc':'white',  
+#                               'boxstyle':'square',
+#                               'alpha':0.8}
+#                                        )
+# 
+# 
+# 
+# 
+#             
+#             # print(STD)
+#             # print(PR)
+#             dias[varkey].add_sample(STD/STD_OBS, PR,
+#                            marker='o', ms=5, ls='',
+#                            #mfc='k', mec='k', # B&W
+#                            mfc=colors[ikey], mec=colors[ikey], # Colors
+#                            label=key)
+#     
+#         # put ticker position, see
+#         # https://matplotlib.org/examples/ticks_and_spines/tick-locators.html 
+#         # dia.ax.axis['bottom'].
+#         # dia.ax.axis['left'].
+#         # dia.ax.axis['left'].
+#     
+#         i += 1
+#     
+#     i = 0
+#     for varkey in ['h','theta','q']:                                                    
+#         for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+#             istation = 0
+#             for icurrent_station,current_station in c4gldata[key].frames['worldmap']['stations'].table.iterrows():
+#                 indices =  (c4gldata[key].frames['stats']['records_all_stations_index'].get_level_values('STNID') == current_station.name)
+#                 station_mod = c4gldata[key].frames['stats']['records_all_stations_mod_stats']['d'+varkey+'dt'].iloc[indices]
+#                 station_obs = c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt'].iloc[indices]
+#     
+#                 axes[varkey].scatter(station_obs,station_mod,marker=symbols[istation],color=colors[ikey])
+#                          #  label=key+", "+\
+#                          #                    'R = '+str(round(PR[0],3))+', '+\
+#                          #                    'RMSE = '+str(round(RMSE,5))+', '+\
+#                          #                    'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+#     
+#     
+#     
+#             # # pl.scatter(obs,mod,label=key+", "+\
+#             # #                              'R = '+str(round(PR[0],3))+', '+\
+#             # #                              'RMSE = '+str(round(RMSE,5))+', '+\
+#             # #                              'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+#                 
+#                 dias[varkey].add_sample(station_mod.std()/station_obs.std(),
+#                                pearsonr(station_mod,station_obs)[0],#annotate=symbols[istation],
+#                                marker=symbols[istation], ms=5, ls='',
+#                                mfc='k', mec='k', # B&W
+#                                #mfc=colors[ikey], mec=colors[ikey], # Colors
+#                                label=key)
+# 
+#                 istation += 1
+#     
+#             if varkey == 'q':
+#                 units_final = r'[$g\, kg^{-1}\, h^{-1}$]'
+#             elif varkey == 'theta':
+#                 units_final = r'[$K\, h^{-1}$]'
+#             elif varkey == 'h':
+#                 units_final = r'[$m\, h^{-1}$]'
+#     
+#             axes[varkey].set_xlabel('Observed')     
+#             axes[varkey].set_title(latex['d'+varkey+'dt']+' '+units_final,fontsize=12)                                     
+# 
+# 
+#         # if varkey == 'q':
+#         #     print('get_xlim not working well...STRANGE')
+#         #     limits =  [np.percentile(nani,1),np.percentile(nani,99)]
+#         # else:
+#         #     limits =  [np.percentile(nani,1.0),np.percentile(nani,99.0)]
+# 
+# 
+#         if i==0:                                    
+#             axes[varkey].set_ylabel('Modelled')                                            
+#         i +=1
+#           
+#         axes[varkey].set_aspect('equal')
+#         low  = c4gldata[key].frames['stats']['records_all_stations_mod_stats']['d'+varkey+'dt'].min()
+#         high  = c4gldata[key].frames['stats']['records_all_stations_mod_stats']['d'+varkey+'dt'].max()
+# 
+#         low  = np.min([low,c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt'].min()])
+#         high  = np.max([high,c4gldata[key].frames['stats']['records_all_stations_obs_afternoon_stats']['d'+varkey+'dt'].max()])
+# 
+#         low = low - (high - low)*0.1
+#         high = high + (high - low)*0.1
+#         axes[varkey].set_xlim([low,high])
+#         axes[varkey].set_ylim([low,high])
+#         abline(1,0,axis=axes[varkey])
+#         if varkey == 'q':
+#             ticks = ticker.FuncFormatter(lambda x, pos:
+#                                          '{0:g}'.format(x*1000.))
+#             axes[varkey].xaxis.set_major_formatter(ticks)
+#             axes[varkey].yaxis.set_major_formatter(ticks)
+#     
+#     
+#     # # legend for different forcing simulations (colors)
+#     # ax = fig.add_axes([0.05,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+#     # leg = []
+#     # for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+#     #     leg1, = ax.plot([],colors[ikey]+'s' ,markersize=10)
+#     #     leg.append(leg1)
+#     # ax.axis('off')
+#     # #leg1 =
+#     # ax.legend(leg,list(args.experiments.strip(' ').split(' ')),loc=2,fontsize=10)
+#     
+#     
+#     # legend for different stations (symbols)
+#     ax = fig.add_axes([0.08,-0.02,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+#     leg = []
+#     isymbol = 0
+#     for icurrent_station,current_station in c4gldata[key].frames['worldmap']['stations'].table.iterrows():
+#         leg1, = ax.plot([],'k'+symbols[isymbol] ,markersize=10)
+#         leg.append(leg1)
+#         isymbol += 1
+#     
+#     # symbol for all stations
+#     leg1, = ax.plot([],'ko',markersize=10)
+#     leg.append(leg1)
+#     
+#     
+#     ax.axis('off')
+#     ax.legend(leg,['HUMPPA','BLLAST','GOAMAZON','All'],loc=2,fontsize=10,ncol=4)
+#     
+#     
+#     fig.subplots_adjust(top=0.95,bottom=0.20,left=0.08,right=0.94,hspace=0.28,wspace=0.29)
+#     
+#     
+#     #pl.legend(leglist,('EMI:WOC','EMI:MED','EMI:BEC'),loc=2,fontsize=16,prop={'family':
+#     #figfn = '/user/data/gent/gvo000/gvo00090/D2D/archive/report/iops_eval_report.png'
+#     
+#     if args.figure_filename is not None:
+#         fig.savefig(args.figure_filename,dpi=200); print("Image file written to:",args.figure_filename)
+#         fig.savefig(args.figure_filename.replace('png','pdf')); print("Image file written to:", args.figure_filename)
+#     fig.show()  
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/class4gl/interface/interface_stations.py b/class4gl/interface/interface_stations.py
new file mode 100644
index 0000000..9fc2162
--- /dev/null
+++ b/class4gl/interface/interface_stations.py
@@ -0,0 +1,405 @@
+import numpy as np
+
+import pandas as pd
+import sys
+
+import matplotlib
+matplotlib.use('TkAgg')
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--experiments')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--load_globaldata',default=False) # load the data needed for the interface
+parser.add_argument('--make_figures',default=None)
+parser.add_argument('--figure_filename',default=None)
+parser.add_argument('--tendencies_revised',default=False)
+parser.add_argument('--obs_filter',default="True")
+args = parser.parse_args()
+
+print('Adding python library:',args.c4gl_path_lib)
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import c4gl_interface_soundings,get_record_yaml
+from class4gl import class4gl_input, data_global,class4gl,units
+#from sklearn.metrics import mean_squared_error
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+#import seaborn.apionly as sns
+import pylab as pl
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import kde
+from scipy.stats import pearsonr                                                
+from taylorDiagram import TaylorDiagram
+from matplotlib import ticker
+# import importlib
+# importlib.reload(mpl); importlib.reload(plt); importlib.reload(sns)
+
+
+
+
+
+latex = {}
+latex['dthetadt'] =  r'$d \theta / dt $'
+latex['dqdt'] =      r'$d q / dt $'
+latex['dhdt'] =      r'$d h / dt $'
+
+def abline(slope, intercept,axis):
+    """Plot a line from slope and intercept"""
+    #axis = plt.gca()
+    x_vals = np.array(axis.get_xlim())
+    y_vals = intercept + slope * x_vals
+    axis.plot(x_vals, y_vals, 'k--')
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+
+
+
+# EXPS  =\
+# {
+# 'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_ADV':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'IOPS_ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+# }
+
+if args.load_globaldata:
+    # iniitialize global data
+    globaldata = data_global()
+    # ...  and load initial data pages
+    globaldata.load_datasets(recalc=0)
+else:
+    globaldata = None
+
+c4gldata = {}
+for key in args.experiments.strip(' ').split(' '):
+    
+    c4gldata[key] = c4gl_interface_soundings( \
+                      args.path_experiments+'/'+key+'/',\
+                      args.path_forcing+'/',\
+                      globaldata,\
+                      refetch_records=False,
+                      obs_filter = (args.obs_filter == 'True'),
+                      tendencies_revised = args.tendencies_revised
+                    )
+
+if bool(args.make_figures):
+    fig = plt.figure(figsize=(10,7))   #width,height
+    i = 1                                                                           
+    axes = {}         
+    axes_taylor = {}         
+    
+    #colors = ['r','g','b','m']
+    colors = ['k']
+    symbols = ['^','x','+']
+    dias = {}
+    
+    for varkey in ['h','theta','q']:                                                    
+        axes[varkey] = fig.add_subplot(2,3,i)                                       
+        #axes_taylor[varkey] = fig.add_subplot(2,3,i+3)                                       
+    
+        #print(obs.std())
+        dias[varkey] =  TaylorDiagram(1., srange=[0.0,1.7],fig=fig, rect=(230+i+3),label='Reference')
+        dias[varkey].add_grid(zorder=-100.)
+        dias[varkey]._ax.axis["left"].label.set_text(\
+            "Normalized standard deviation")
+        i += 1
+    i = 1                                                                           
+    for varkey in ['h','theta','q']:                                                    
+        obs = c4gldata[args.experiments.strip().split()[0]].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt']
+        STD_OBS = obs.std()
+        if i == 1:
+            axes[varkey].annotate('Normalized standard deviation',\
+                        xy= (0.05,0.36),
+                        color='black',
+                        rotation=90.,
+                        xycoords='figure fraction',
+                        weight='normal',
+                        fontsize=10.,
+                        horizontalalignment='center',
+                        verticalalignment='center' ,
+                        #bbox={'edgecolor':'black',
+                        #      'boxstyle':'circle',
+                        #      'fc':koeppen.color,
+                        #      'alpha':1.0}
+                       )
+        # dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+        # dias[varkey]._ax.axis["left"].axis.set_major_locator(np.arange(0.,2.,0.25))
+        #dias[varkey]._ax.axis["left"].axis.set_ticks(np.arange(0.,2.,0.25))
+        # Q95 = obs.quantile(0.95)
+        # Q95 = obs.quantile(0.90)
+        # Add RMS contours, and label them
+        contours = dias[varkey].add_contours(levels=5, colors='0.7') # 5 levels
+        dias[varkey].ax.clabel(contours, inline=1, fontsize=10,fmt='%.1f')
+        #dia._ax.set_title(season.capitalize())
+        i += 1
+    
+        for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+            mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt']
+            obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt']
+            x, y = obs.values,mod.values
+            print(key,len(obs.values))
+    
+            #scores
+            PR = pearsonr(mod,obs)[0]
+            RMSE = rmse(obs,mod)                                               
+            BIAS = np.mean(mod) - np.mean(obs)
+            STD = mod.std()
+
+            # print(STD)
+            # print(PR)
+            print(varkey,STD,STD_OBS,STD/STD_OBS,PR)
+            dias[varkey].add_sample(STD/STD_OBS, PR,
+                           marker='o', ms=7, ls='',
+                           #mfc='k', mec='k', # B&W
+                           mfc=colors[ikey], mec=colors[ikey], # Colors
+                           label=key,zorder=101)
+                
+            fit = np.polyfit(x,y,deg=1)
+
+            if varkey == 'q':
+                axes[varkey].plot(x, fit[0] * x + fit[1],\
+                                  color=colors[ikey],alpha=0.8,lw=2,\
+                                  label=key+", "+\
+                           'RMSE = '+format((RMSE*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+ '\n'+\
+                           'Bias = '+format((BIAS*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+' \n'+\
+                           r'$R$ = '+format(PR,'0.2f') )
+
+
+            elif varkey == 'h':
+                axes[varkey].plot(x, fit[0] * x + fit[1],\
+                                  color=colors[ikey],alpha=0.8,lw=2,\
+                                  label=key+", "+\
+                            'RMSE = '+format(RMSE,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+                            'Bias = '+format(BIAS,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+                            r'$R$ = '+format(PR,'0.2f'))
+            else: #theta
+                axes[varkey].plot(x, fit[0] * x + fit[1],\
+                                  color=colors[ikey],alpha=0.8,lw=2,\
+                                  label=key+", "+\
+                            'RMSE = '+format(RMSE,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+                            'Bias = '+format(BIAS,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+                            r'$R$ = '+format(PR,'0.2f'))
+
+            if varkey == 'q':
+                annotate_text = \
+                               'RMSE = '+format((RMSE*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+ '\n'+\
+                               'Bias = '+format((BIAS*1000.),'0.2f')+r'$\,  \mathrm{g\,  kg^{-1}\,  h^{-1}}$'+' \n'+\
+                               r'$R$ = '+format(PR,'0.2f')
+                ann = axes[varkey].annotate(annotate_text, xy=(0.95, .05 ), xycoords='axes fraction',fontsize=9,
+                # ann = axes[varkey].annotate(annotate_text, xy=(0.05, .97 ), xycoords='axes fraction',fontsize=9,
+       horizontalalignment='right', verticalalignment='bottom' ,
+        bbox={'edgecolor':'black',
+                          'fc':'white',  
+                              'boxstyle':'square',
+                              'alpha':0.8}
+                                       )
+            elif varkey == 'h':
+                annotate_text = \
+                                'RMSE = '+format(RMSE,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+                                'Bias = '+format(BIAS,'0.1f')+r'$\,  \mathrm{m\, h^{-1}}$'+'\n'+\
+                                r'$R$ = '+format(PR,'0.2f')
+                ann = axes[varkey].annotate(annotate_text, xy=(0.95, .05 ), xycoords='axes fraction',fontsize=9,
+       horizontalalignment='right', verticalalignment='bottom' ,
+        bbox={'edgecolor':'black',
+                          'fc':'white',  
+                              'boxstyle':'square',
+                              'alpha':0.8}
+                                       )
+            else:
+                annotate_text = \
+                                'RMSE = '+format(RMSE,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+                                'Bias = '+format(BIAS,'0.3f')+r'$\, \mathrm{K\, h^{-1}}$'+'\n'+\
+                                r'$R$ = '+format(PR,'0.2f')
+
+                ann = axes[varkey].annotate(annotate_text, xy=(0.95, .05 ), xycoords='axes fraction',fontsize=9,
+       horizontalalignment='right', verticalalignment='bottom' ,
+        bbox={'edgecolor':'black',
+                          'fc':'white',  
+                              'boxstyle':'square',
+                              'alpha':0.8}
+                                       )
+
+
+
+
+            
+    
+        # put ticker position, see
+        # https://matplotlib.org/examples/ticks_and_spines/tick-locators.html 
+        # dia.ax.axis['bottom'].
+        # dia.ax.axis['left'].
+        # dia.ax.axis['left'].
+    
+        i += 1
+    
+    i = 0
+    for varkey in ['h','theta','q']:                                                    
+        for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+            istation = 0
+            for icurrent_station,current_station in c4gldata[key].frames['worldmap']['stations'].table.iterrows():
+                indices =  (c4gldata[key].frames['stats']['records_all_stations_index'].get_level_values('STNID') == current_station.name)
+                station_end_mod = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt'].iloc[indices]
+                station_obs = c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt'].iloc[indices]
+    
+                axes[varkey].scatter(station_obs,station_end_mod,marker=symbols[istation],color=colors[ikey])
+                         #  label=key+", "+\
+                         #                    'R = '+str(round(PR[0],3))+', '+\
+                         #                    'RMSE = '+str(round(RMSE,5))+', '+\
+                         #                    'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+    
+    
+    
+            # # pl.scatter(obs,mod,label=key+", "+\
+            # #                              'R = '+str(round(PR[0],3))+', '+\
+            # #                              'RMSE = '+str(round(RMSE,5))+', '+\
+            # #                              'BIAS = '+str(round(BIAS,5)),s=1.,color=colors[ikey])
+                
+                dias[varkey].add_sample(station_end_mod.std()/station_obs.std(),
+                               pearsonr(station_end_mod,station_obs)[0],#annotate=symbols[istation],
+                               marker=symbols[istation], ms=7, ls='',
+                               mfc='k', mec='k', # B&W
+                               #mfc=colors[ikey], mec=colors[ikey], # Colors
+                               label=key,zorder=100)
+
+                istation += 1
+    
+            if varkey == 'q':
+                units_final = r'[$g\, kg^{-1}\, h^{-1}$]'
+            elif varkey == 'theta':
+                units_final = r'[$K\, h^{-1}$]'
+            elif varkey == 'h':
+                units_final = r'[$m\, h^{-1}$]'
+    
+            axes[varkey].set_xlabel('Observed')     
+            axes[varkey].set_title(latex['d'+varkey+'dt']+' '+units_final,fontsize=12)                                     
+
+
+        # if varkey == 'q':
+        #     print('get_xlim not working well...STRANGE')
+        #     limits =  [np.percentile(nani,1),np.percentile(nani,99)]
+        # else:
+        #     limits =  [np.percentile(nani,1.0),np.percentile(nani,99.0)]
+
+
+        if i==0:                                    
+            axes[varkey].set_ylabel('Modelled')                                            
+        i +=1
+          
+        axes[varkey].set_aspect('equal')
+        low  = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt'].min()
+        high  = c4gldata[key].frames['stats']['records_all_stations_end_mod_stats']['d'+varkey+'dt'].max()
+
+        low  = np.min([low,c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt'].min()])
+        high  = np.max([high,c4gldata[key].frames['stats']['records_all_stations_end_obs_stats']['d'+varkey+'dt'].max()])
+
+        low = low - (high - low)*0.1
+        high = high + (high - low)*0.1
+        axes[varkey].set_xlim([low,high])
+        axes[varkey].set_ylim([low,high])
+        abline(1,0,axis=axes[varkey])
+        if varkey == 'q':
+            ticks = ticker.FuncFormatter(lambda x, pos:
+                                         '{0:g}'.format(x*1000.))
+            axes[varkey].xaxis.set_major_formatter(ticks)
+            axes[varkey].yaxis.set_major_formatter(ticks)
+    
+    
+    # # legend for different forcing simulations (colors)
+    # ax = fig.add_axes([0.05,0.00,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    # leg = []
+    # for ikey,key in enumerate(args.experiments.strip(' ').split(' ')):
+    #     leg1, = ax.plot([],colors[ikey]+'s' ,markersize=10)
+    #     leg.append(leg1)
+    # ax.axis('off')
+    # #leg1 =
+    # ax.legend(leg,list(args.experiments.strip(' ').split(' ')),loc=2,fontsize=10)
+    
+    
+    # legend for different stations (symbols)
+    ax = fig.add_axes([0.08,-0.02,0.15,0.15]) #[*left*, *bottom*, *width*,    *height*]
+    leg = []
+    isymbol = 0
+    for icurrent_station,current_station in c4gldata[key].frames['worldmap']['stations'].table.iterrows():
+        leg1, = ax.plot([],'k'+symbols[isymbol] ,markersize=14)
+        leg.append(leg1)
+        isymbol += 1
+    
+    # symbol for all stations
+    leg1, = ax.plot([],'ko',markersize=14)
+    leg.append(leg1)
+    
+    
+    ax.axis('off')
+    ax.legend(leg,['HUMPPA','BLLAST','GOAMAZON','All'],loc=2,fontsize=10,ncol=4)
+    
+    
+    fig.subplots_adjust(top=0.95,bottom=0.20,left=0.08,right=0.94,hspace=0.28,wspace=0.29)
+    
+    
+    #pl.legend(leglist,('EMI:WOC','EMI:MED','EMI:BEC'),loc=2,fontsize=16,prop={'family':
+    #figfn = '/user/data/gent/gvo000/gvo00090/D2D/archive/report/iops_eval_report.png'
+    
+    if args.figure_filename is not None:
+        fig.savefig(args.figure_filename,dpi=200); print("Image file written to:",args.figure_filename)
+        fig.savefig(args.figure_filename.replace('png','pdf')); print("Image file written to:", args.figure_filename)
+    fig.show()  
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/class4gl/interface/taylorDiagram.py b/class4gl/interface/taylorDiagram.py
new file mode 100644
index 0000000..8d3e72d
--- /dev/null
+++ b/class4gl/interface/taylorDiagram.py
@@ -0,0 +1,259 @@
+#!/usr/bin/env python
+# Copyright: This document has been placed in the public domain.
+
+"""
+Taylor diagram (Taylor, 2001) implementation.
+"""
+
+__version__ = "Time-stamp: <2018-05-17 19:41:54 ycopin>"
+__author__ = "Yannick Copin "
+
+import numpy as NP
+import matplotlib.pyplot as PLT
+from matplotlib import ticker
+
+
+class TaylorDiagram(object):
+    """
+    Taylor diagram.
+
+    Plot model standard deviation and correlation to reference (data)
+    sample in a single-quadrant polar plot, with r=stddev and
+    theta=arccos(correlation).
+    """
+
+    def __init__(self, refstd,
+                 fig=None, rect=111, label='_', srange=(0, 1.5), extend=False):
+        """
+        Set up Taylor diagram axes, i.e. single quadrant polar
+        plot, using `mpl_toolkits.axisartist.floating_axes`.
+
+        Parameters:
+
+        * refstd: reference standard deviation to be compared to
+        * fig: input Figure or None
+        * rect: subplot definition
+        * label: reference label
+        * srange: stddev axis extension, in units of *refstd*
+        * extend: extend diagram to negative correlations
+        """
+
+        from matplotlib.projections import PolarAxes
+        import mpl_toolkits.axisartist.floating_axes as FA
+        import mpl_toolkits.axisartist.grid_finder as GF
+
+        self.refstd = refstd            # Reference standard deviation
+
+        tr = PolarAxes.PolarTransform()
+
+        # Correlation labels
+        rlocs = NP.array([0, 0.2, 0.4, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1])
+        if extend:
+            # Diagram extended to negative correlations
+            self.tmax = NP.pi
+            rlocs = NP.concatenate((-rlocs[:0:-1], rlocs))
+        else:
+            # Diagram limited to positive correlations
+            self.tmax = NP.pi/2
+        tlocs = NP.arccos(rlocs)        # Conversion to polar angles
+        gl1 = GF.FixedLocator(tlocs)    # Positions
+        tf1 = GF.DictFormatter(dict(zip(tlocs, map(str, rlocs))))
+
+        # Standard deviation axis extent (in units of reference stddev)
+        self.smin = srange[0] * self.refstd
+        self.smax = srange[1] * self.refstd
+
+        ghelper = FA.GridHelperCurveLinear(
+            tr,
+            extremes=(0, self.tmax, self.smin, self.smax),
+            grid_locator1=gl1, tick_formatter1=tf1)
+
+        if fig is None:
+            fig = PLT.figure()
+
+        ax = FA.FloatingSubplot(fig, rect, grid_helper=ghelper)
+        fig.add_subplot(ax)
+
+        # Adjust axes
+        ax.axis["top"].set_axis_direction("bottom")   # "Angle axis"
+        ax.axis["top"].toggle(ticklabels=True, label=True)
+        ax.axis["top"].major_ticklabels.set_axis_direction("top")
+        ax.axis["top"].label.set_axis_direction("top")
+        ax.axis["top"].label.set_text("Correlation")
+
+        ax.axis["left"].set_axis_direction("bottom")  # "X axis"
+        #ax.axis["left"].label.set_text("Standard deviation (model)/ Observed (observations)")
+
+
+        # #ax.axis["left"].axis.set_ticklabels(["1.0","0.8","0.6","0.4","0.2","0.0","0.2","0.4","0.6","0.8","1.0"])
+        # ticks = ticker.FuncFormatter(lambda x, pos: '{0:g}'.format(x-1.))
+        # ax.axis["left"].axis.set_major_formatter(ticks)
+
+        ax.axis["left"].toggle(ticklabels=False, label=False)
+        ax.axis["right"].set_axis_direction("top")    # "Y-axis"
+        ax.axis["right"].toggle(ticklabels=True)
+        ax.axis["right"].major_ticklabels.set_axis_direction(
+            "bottom" if extend else "left")
+
+        ax.axis["bottom"].set_visible(False)          # Unused
+
+        self._ax = ax                   # Graphical axes
+        self.ax = ax.get_aux_axes(tr)   # Polar coordinates
+        # # DOESNT WORK!!!!
+        # ax.axes.set_xticks([0.,0.2,0.4,1.])
+        # ax.axes.set_yticks([0.,0.2,0.4,1.])
+
+        # Add reference point and stddev contour
+        l, = self.ax.plot([0], self.refstd, 'k*',
+                          ls='', ms=10, label=label)
+        t = NP.linspace(0, self.tmax)
+        r = NP.zeros_like(t) + self.refstd
+        self.ax.plot(t, r, 'k--', label='_')
+
+        # Collect sample points for latter use (e.g. legend)
+        self.samplePoints = [l]
+
+    def add_sample(self, stddev, corrcoef, annotate=None,*args, **kwargs):
+        """
+        Add sample (*stddev*, *corrcoeff*) to the Taylor
+        diagram. *args* and *kwargs* are directly propagated to the
+        `Figure.plot` command.
+        """
+        print(annotate)
+
+        if annotate is not None:
+            l = self.ax.annotate(annotate,xy=(NP.arccos(corrcoef), stddev),\
+                              *args, **kwargs)  # (theta, radius)
+        else:
+            l, = self.ax.plot(NP.arccos(corrcoef), stddev,
+                          *args, **kwargs)  # (theta, radius)
+        self.samplePoints.append(l)
+
+        return l
+
+    def add_grid(self, *args, **kwargs):
+        """Add a grid."""
+
+        self._ax.grid(*args, **kwargs)
+
+    def add_contours(self, levels=5, **kwargs):
+        """
+        Add constant centered RMS difference contours, defined by *levels*.
+        """
+
+        rs, ts = NP.meshgrid(NP.linspace(self.smin, self.smax),
+                             NP.linspace(0, self.tmax))
+        # Compute centered RMS difference
+        rms = NP.sqrt(self.refstd**2 + rs**2 - 2*self.refstd*rs*NP.cos(ts))
+
+        contours = self.ax.contour(ts, rs, rms, levels, **kwargs)
+
+        return contours
+
+
+def test1():
+    """Display a Taylor diagram in a separate axis."""
+
+    # Reference dataset
+    x = NP.linspace(0, 4*NP.pi, 100)
+    data = NP.sin(x)
+    refstd = data.std(ddof=1)           # Reference standard deviation
+
+    # Generate models
+    m1 = data + 0.2*NP.random.randn(len(x))     # Model 1
+    m2 = 0.8*data + .1*NP.random.randn(len(x))  # Model 2
+    m3 = NP.sin(x-NP.pi/10)                     # Model 3
+
+    # Compute stddev and correlation coefficient of models
+    samples = NP.array([ [m.std(ddof=1), NP.corrcoef(data, m)[0, 1]]
+                         for m in (m1, m2, m3)])
+
+    fig = PLT.figure(figsize=(10, 4))
+
+    ax1 = fig.add_subplot(1, 2, 1, xlabel='X', ylabel='Y')
+    # Taylor diagram
+    dia = TaylorDiagram(refstd, fig=fig, rect=122, label="Reference")
+
+    colors = PLT.matplotlib.cm.jet(NP.linspace(0, 1, len(samples)))
+
+    ax1.plot(x, data, 'ko', label='Data')
+    for i, m in enumerate([m1, m2, m3]):
+        ax1.plot(x, m, c=colors[i], label='Model %d' % (i+1))
+    ax1.legend(numpoints=1, prop=dict(size='small'), loc='best')
+
+    # Add the models to Taylor diagram
+    for i, (stddev, corrcoef) in enumerate(samples):
+        dia.add_sample(stddev, corrcoef,
+                       marker='$%d$' % (i+1), ms=10, ls='',
+                       mfc=colors[i], mec=colors[i],
+                       label="Model %d" % (i+1))
+
+    # Add grid
+    dia.add_grid()
+
+    # Add RMS contours, and label them
+    print('BLABLA')
+    # contours = dia.add_contours(colors='lightgrey')
+    PLT.clabel(contours, inline=1, fontsize=10, fmt='%.2f')
+
+    # Add a figure legend
+    fig.legend(dia.samplePoints,
+               [ p.get_label() for p in dia.samplePoints ],
+               numpoints=1, prop=dict(size='small'), loc='upper right')
+
+    return dia
+
+
+def test2():
+    """
+    Climatology-oriented example (after iteration w/ Michael A. Rawlins).
+    """
+
+    # Reference std
+    stdref = 48.491
+
+    # Samples std,rho,name
+    samples = [[25.939, 0.385, "Model A"],
+               [29.593, 0.509, "Model B"],
+               [33.125, 0.585, "Model C"],
+               [29.593, 0.509, "Model D"],
+               [71.215, 0.473, "Model E"],
+               [27.062, 0.360, "Model F"],
+               [38.449, 0.342, "Model G"],
+               [35.807, 0.609, "Model H"],
+               [17.831, 0.360, "Model I"]]
+
+    fig = PLT.figure()
+
+    dia = TaylorDiagram(stdref, fig=fig, label='Reference', extend=True)
+    dia.samplePoints[0].set_color('r')  # Mark reference point as a red star
+
+    # Add models to Taylor diagram
+    for i, (stddev, corrcoef, name) in enumerate(samples):
+        dia.add_sample(stddev, corrcoef,
+                       marker='$%d$' % (i+1), ms=10, ls='',
+                       mfc='k', mec='k',
+                       label=name)
+
+    # Add RMS contours, and label them
+    # contours = dia.add_contours(levels=5, colors='lightgrey')  # 5 levels in grey
+    PLT.clabel(contours, inline=1, fontsize=10, fmt='%.0f')
+
+    dia.add_grid()                                  # Add grid
+    dia._ax.axis[:].major_ticks.set_tick_out(True)  # Put ticks outward
+
+    # Add a figure legend and title
+    fig.legend(dia.samplePoints,
+               [ p.get_label() for p in dia.samplePoints ],
+               numpoints=1, prop=dict(size='small'), loc='upper right')
+    # fig.suptitle("Taylor diagram", size='x-large')  # Figure title
+
+    return dia
+
+
+if __name__ == '__main__':
+
+    dia = test1()
+    dia = test2()
+
+    PLT.show()
diff --git a/class4gl/interface/test.test b/class4gl/interface/test.test
new file mode 100644
index 0000000..969d1d3
--- /dev/null
+++ b/class4gl/interface/test.test
@@ -0,0 +1,276 @@
+---
+# CLASS4GL input; format version: 0.1
+index: 4
+pars:
+  Ammax298: [2.2, 1.7]
+  CO2: 400.8322760704748
+  CO22_h: null
+  CO2comp298: [68.5, 4.3]
+  CO2tend: 0.0
+  Cm: 0.014889170302298154
+  Cs: 0.012038840098238213
+  Cw: 0.0016
+  E0: 53300.0
+  G: 47.29774110009936
+  H: 190.5623007925413
+  Kx: [0.7, 0.7]
+  L: -6.853794630136477
+  LE: 214.88314861335857
+  LEliq: 0.0
+  LEpot: 319.34762471715567
+  LEref: 243.84639777941322
+  LEsoil: 43.588158263249724
+  LEveg: 171.29499035010883
+  Lwin: 307.8635861165938
+  Lwout: 426.33896646668103
+  M: 0.0
+  P_h: 82175.50598724303
+  Q: 452.7431905059985
+  Q10Am: [2.0, 2.0]
+  Q10CO2: [1.5, 1.5]
+  Q10gm: [2.0, 2.0]
+  R10: 0.23
+  RH_h: 1.674122332852863
+  Rib: -6.70652127177839
+  Swin: 739.103621663443
+  Swout: 167.8850508073574
+  T1Am: [281.0, 286.0]
+  T1gm: [278.0, 286.0]
+  T2: 284.68364969889325
+  T2Am: [311.0, 311.0]
+  T2gm: [301.0, 309.0]
+  T2m: 282.3123067089991
+  T_h: 274.8936229768578
+  Ts: 294.47523957923886
+  Tsoil: 286.4586732910864
+  ac: 0.0
+  ad: [0.07, 0.15]
+  advCO2: 0.0
+  advt: 1.0355662748843815e-05
+  alpha0: [0.017, 0.014]
+  c3c4: c3
+  c_beta: 0
+  dCO2: -22.832276070474784
+  dCO2tend: 0.0
+  dFz: 0.0
+  divU: 0.0
+  dq: -0.003052156485017018
+  dtcur: 60.0
+  dtheta: 0.1
+  dthetav: -0.4365906447159773
+  dtmax: 1598.8514968081563
+  du: -0.4229086217862608
+  dv: 0.28673977724043115
+  dz: 50.0
+  dz_h: 50
+  dztend: -0.1395839151513645
+  e: 1388.9934836777713
+  e2m: 952.6963178543036
+  esat: 1731.8968600367527
+  esat2m: 1160.2806224411495
+  f0: [0.89, 0.85]
+  firsttime: true
+  gammaCO2: 0.0
+  gammaq: -2.2753052840179227e-05
+  gammatheta: 0.002144498648676304
+  gammau: 0.028469046551934885
+  gammav: 0.017160662703889926
+  gm298: [7.0, 17.5]
+  gmin: [0.00025, 0.00025]
+  h: 1383.2363985770135
+  lcl: 428.232209487189
+  ls_type: js
+  mair: 28.9
+  mco2: 44.0
+  nuco2q: 1.6
+  q: 0.008771106059366231
+  q2_h: null
+  q2m: 0.006016011266044435
+  qsat: 0.010936445146628022
+  qsurf: 0.011939539970569558
+  ra: 38.47545242647453
+  rs: 49.409679791439665
+  sp: 99023.24348958333
+  substep: false
+  substeps: 0
+  sw_ac: [adv]
+  sw_ap: true
+  sw_cu: false
+  sw_fixft: false
+  sw_lit: false
+  sw_ml: true
+  test: 0.0
+  testing: 0.0
+  theta: 288.39566185252
+  thetasurf: 294.472849715022
+  thetav: 289.93868670419346
+  thetavsurf: 296.61753063426823
+  time: 12.116666666666667
+  tsteps: 360
+  u: 0.817076520336863
+  u2m: 0.8170765203368628
+  ustar: 0.26343099197090714
+  uw: -0.026264240506899815
+  v: 0.04516931051262895
+  v2m: 0.045169310512628943
+  vw: -0.0014519296605725784
+  wCO2: 0.0
+  wCO2A: 0
+  wCO2M: 0
+  wCO2R: 0
+  wCO2e: 0.0
+  wf: 0.0
+  wg: 0.35105639595370375
+  wmax: 0.55
+  wmin: 0.005
+  wq: 7.162771620445285e-05
+  wqM: 0.0
+  wqe: 0.0
+  wstar: 1.9991113910017524
+  wtheta: 0.15801185803693307
+  wthetae: -0.0
+  wthetav: 0.17070715833083436
+  wthetave: -0.03414143166616687
+  z0h: 0.0
+  z0m: 0.0
+  zeta: -20.182052034282645
+  zlcl: 428.232209487189
+  zslz0m: 480.2904085186834
+---
+# CLASS4GL input; format version: 0.1
+index: 2944
+pars:
+  Ammax298: [2.2, 1.7]
+  CO2: 400.8322760704748
+  CO22_h: null
+  CO2comp298: [68.5, 4.3]
+  CO2tend: 0.0
+  Cm: 0.014889170302298154
+  Cs: 0.012038840098238213
+  Cw: 0.0016
+  E0: 53300.0
+  G: 47.29774110009936
+  H: 190.5623007925413
+  Kx: [0.7, 0.7]
+  L: -6.853794630136477
+  LE: 214.88314861335857
+  LEliq: 0.0
+  LEpot: 319.34762471715567
+  LEref: 243.84639777941322
+  LEsoil: 43.588158263249724
+  LEveg: 171.29499035010883
+  Lwin: 307.8635861165938
+  Lwout: 426.33896646668103
+  M: 0.0
+  P_h: 82175.50598724303
+  Q: 452.7431905059985
+  Q10Am: [2.0, 2.0]
+  Q10CO2: [1.5, 1.5]
+  Q10gm: [2.0, 2.0]
+  R10: 0.23
+  RH_h: 1.674122332852863
+  Rib: -6.70652127177839
+  Swin: 739.103621663443
+  Swout: 167.8850508073574
+  T1Am: [281.0, 286.0]
+  T1gm: [278.0, 286.0]
+  T2: 284.68364969889325
+  T2Am: [311.0, 311.0]
+  T2gm: [301.0, 309.0]
+  T2m: 282.3123067089991
+  T_h: 274.8936229768578
+  Ts: 294.47523957923886
+  Tsoil: 286.4586732910864
+  ac: 0.0
+  ad: [0.07, 0.15]
+  advCO2: 0.0
+  advt: 1.0355662748843815e-05
+  alpha0: [0.017, 0.014]
+  c3c4: c3
+  c_beta: 0
+  dCO2: -22.832276070474784
+  dCO2tend: 0.0
+  dFz: 0.0
+  divU: 0.0
+  dq: -0.003052156485017018
+  dtcur: 60.0
+  dtheta: 0.1
+  dthetav: -0.4365906447159773
+  dtmax: 1598.8514968081563
+  du: -0.4229086217862608
+  dv: 0.28673977724043115
+  dz: 50.0
+  dz_h: 50
+  dztend: -0.1395839151513645
+  e: 1388.9934836777713
+  e2m: 952.6963178543036
+  esat: 1731.8968600367527
+  esat2m: 1160.2806224411495
+  f0: [0.89, 0.85]
+  firsttime: true
+  gammaCO2: 0.0
+  gammaq: -2.2753052840179227e-05
+  gammatheta: 0.002144498648676304
+  gammau: 0.028469046551934885
+  gammav: 0.017160662703889926
+  gm298: [7.0, 17.5]
+  gmin: [0.00025, 0.00025]
+  h: 1383.2363985770135
+  lcl: 428.232209487189
+  ls_type: js
+  mair: 28.9
+  mco2: 44.0
+  nuco2q: 1.6
+  q: 0.008771106059366231
+  q2_h: null
+  q2m: 0.006016011266044435
+  qsat: 0.010936445146628022
+  qsurf: 0.011939539970569558
+  ra: 38.47545242647453
+  rs: 49.409679791439665
+  sp: 99023.24348958333
+  substep: false
+  substeps: 0
+  sw_ac: [adv]
+  sw_ap: true
+  sw_cu: false
+  sw_fixft: false
+  sw_lit: false
+  sw_ml: true
+  test: 0.0
+  testing: 0.0
+  theta: 288.39566185252
+  thetasurf: 294.472849715022
+  thetav: 289.93868670419346
+  thetavsurf: 296.61753063426823
+  time: 12.116666666666667
+  tsteps: 360
+  u: 0.817076520336863
+  u2m: 0.8170765203368628
+  ustar: 0.26343099197090714
+  uw: -0.026264240506899815
+  v: 0.04516931051262895
+  v2m: 0.045169310512628943
+  vw: -0.0014519296605725784
+  wCO2: 0.0
+  wCO2A: 0
+  wCO2M: 0
+  wCO2R: 0
+  wCO2e: 0.0
+  wf: 0.0
+  wg: 0.35105639595370375
+  wmax: 0.55
+  wmin: 0.005
+  wq: 7.162771620445285e-05
+  wqM: 0.0
+  wqe: 0.0
+  wstar: 1.9991113910017524
+  wtheta: 0.15801185803693307
+  wthetae: -0.0
+  wthetav: 0.17070715833083436
+  wthetave: -0.03414143166616687
+  z0h: 0.0
+  z0m: 0.0
+  zeta: -20.182052034282645
+  zlcl: 428.232209487189
+  zslz0m: 480.2904085186834
diff --git a/class4gl/interface/test_histogram.py b/class4gl/interface/test_histogram.py
new file mode 100644
index 0000000..7579a9c
--- /dev/null
+++ b/class4gl/interface/test_histogram.py
@@ -0,0 +1,25 @@
+
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+
+data = [220,14.2,150,400,420]
+error = [10, 1, 20, 60, 10]
+x = [i + .5 for i in range(5)]
+
+fig, ax = plt.subplots()
+bar = ax.bar(x, data,0.1 +np.arange(len(data))*0.9/len(data), align="center", yerr=error)
+plot = ax.plot(x, data)
+ax.set_xticks(x)
+ax.set_xticklabels(('wt', 'N23PP', 'N23PP/PEKN', 'PEKN', 'N23PP/PEKN/L28F'))
+ax.set_title(r"Everything in the document can use m$\alpha$th language",
+             y=1.05)
+ax.set_ylabel(r"Rate (s$^{-1}$)", labelpad=10)
+ax.set_xlabel("Mutant",labelpad=10)
+ax.yaxis.set_ticks_position('left')
+ax.xaxis.set_ticks_position('bottom')
+plt.savefig('test.png')
+plt.show()
+
+
diff --git a/class4gl/interface/world_histogram.py b/class4gl/interface/world_histogram.py
new file mode 100644
index 0000000..b1fcec6
--- /dev/null
+++ b/class4gl/interface/world_histogram.py
@@ -0,0 +1,367 @@
+'''
+import numpy as np
+import pandas as pd
+import sys
+import matplotlib
+matplotlib.use('TkAgg')
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--experiments')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--load_globaldata',default=False)
+parser.add_argument('--make_figures',default=None)
+parser.add_argument('--show_control_parameters',default=True)
+parser.add_argument('--figure_filename',default=None)
+parser.add_argument('--figure_filename_2',default=None)
+parser.add_argument('--experiments_labels',default=None)
+parser.add_argument('--obs_filter',default='True')
+args = parser.parse_args()
+
+print('Adding python library:',args.c4gl_path_lib)
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import c4gl_interface_soundings,get_record_yaml
+from class4gl import class4gl_input, data_global,class4gl,units
+#from sklearn.metrics import mean_squared_error
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+import seaborn.apionly as sns
+import pylab as pl
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import kde
+from scipy.stats import pearsonr                                                
+from taylorDiagram import TaylorDiagram
+from matplotlib import ticker
+import xarray as xr
+# import importlib
+# importlib.reload(mpl); importlib.reload(plt); importlib.reload(sns)
+
+
+if args.experiments_labels is None:
+    keylabels = args.experiments.strip().split(' ')
+else:
+    keylabels = args.experiments_labels.strip().split(';')
+
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+
+
+
+# EXPS  =\
+# {
+# 'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'GLOBAL_ITER_ADV':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+# #'IOPS_ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+# # 'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+# }
+
+if bool(args.load_globaldata):
+    # iniitialize global data
+    globaldata = data_global()
+    # ...  and load initial data pages
+    globaldata.load_datasets(recalc=0)
+else:
+    globaldata = None
+
+c4gldata = {}
+for key in args.experiments.strip(' ').split(' '):
+    
+    c4gldata[key] = c4gl_interface_soundings( \
+                      args.path_experiments+'/'+key+'/',\
+                      args.path_forcing+'/',\
+                      globaldata,\
+                      refetch_records=False,
+                      obs_filter = (args.obs_filter == 'True')
+                                            
+                    )
+'''
+
+# kgccolors = {
+#     'Dfa':['navy','white'],
+#     'Cfb':['green','white']       ,
+#     'BSk':['tan','black']      ,
+#     'Csb':['lightgreen','black'] ,     
+#     'Cfa':['darkgreen','white']  ,    
+#     'BWh':['orange','black']      ,
+#     'Aw' :['pink','black'],
+#     'Dwc':['rebeccapurple','white'] ,    
+#     'Dfb':['darkviolet','white']    , 
+# }
+
+
+
+
+
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib import rc
+import numpy as np
+import seaborn as sns
+# activate latex text rendering
+sns.reset_orig()
+
+data = [220,14.2,150,400,420,100,150,30,60,20,500,]
+error = [10, 1, 20, 60, 10,10, 1, 20, 60, 10, 5, ]
+
+ini = c4gldata['GLOBAL_ADV'].frames['stats']['records_all_stations_ini'].set_index(['STNID','dates'])
+vals = c4gldata['GLOBAL_ADV'].frames['stats']['records_all_stations_mod']
+stats = c4gldata['GLOBAL_ADV'].frames['stats']['records_all_stations_mod_stats']
+
+ini_fc = c4gldata['GLOBAL_ADV_FC'].frames['stats']['records_all_stations_ini'].set_index(['STNID','dates'])
+vals_fc = c4gldata['GLOBAL_ADV_FC'].frames['stats']['records_all_stations_mod']
+stats_fc = c4gldata['GLOBAL_ADV_FC'].frames['stats']['records_all_stations_mod']
+
+vals_obs = c4gldata['GLOBAL_ADV'].frames['stats']['records_all_stations_obs_afternoon']
+stats_obs = c4gldata['GLOBAL_ADV'].frames['stats']['records_all_stations_obs_afternoon']
+
+ini_common_index = ini.index.intersection(ini_fc.index)
+
+vals.index = ini.index
+vals = vals.loc[ini_common_index]
+
+stats.index = ini.index
+stats = stats.loc[ini_common_index]
+
+
+vals_obs.index = ini.index
+vals_obs = vals_obs.loc[ini_common_index]
+
+stats_obs.index = ini.index
+stats_obs = stats_obs.loc[ini_common_index]
+
+
+ini = ini.loc[ini_common_index]
+
+
+vals_fc.index = ini_fc.index
+vals_fc = vals_fc.loc[ini_common_index]
+
+stats_fc.index = ini_fc.index
+stats_fc = stats_fc.loc[ini_common_index]
+
+ini_fc = ini_fc.loc[ini_common_index]
+
+dlat = 10
+blat = np.arange(-55.,60.,dlat)[[2,3,4,7,8,9,10,11]]
+lats = (blat[1:] + blat[:-1])/2.
+
+fig = plt.figure(figsize=(10,6)) 
+variables = ['h','theta',r'q']
+labels = [r'$h$',r'$\theta$',r'$q$']
+units = ['m','K','g/kg']
+xlims = [(-10,3000),(280,310.),(2.5,17.5)]
+#var = "theta"
+for ivar,var in enumerate(variables):
+    ax = fig.add_subplot(1,len(variables),ivar+1,)
+    data = []
+    data_025 = []
+    data_075 = []
+    
+    data_fc = []
+    data_fc_025 = []
+    data_fc_075 = []
+    
+    data_obs = []
+    data_obs_025 = []
+    data_obs_075 = []
+    
+    data_ini = []
+    data_ini_025 = []
+    data_ini_075 = []
+    
+    lnts = []
+    
+    for ilat,lat in enumerate(lats):
+        print(ilat,lat)
+    # 
+        query = 'latitude >= '+str(blat[ilat])+' and '+ 'latitude < '+str(blat[ilat+1])
+        print(query)
+        select = ini.query(query)
+        if len(select) >= 7:
+            lnts.append(len(select))
+            #print(stats.iloc[select.index])
+            print(stats.loc[select.index])
+            data.append(vals.loc[select.index][var].mean()) 
+            data_025.append(vals.loc[select.index][var].quantile(0.25)) 
+            data_075.append(vals.loc[select.index][var].quantile(0.75)) 
+            data_fc.append(vals_fc.loc[select.index][var].mean()) 
+            data_fc_025.append(vals_fc.loc[select.index][var].quantile(0.25)) 
+            data_fc_075.append(vals_fc.loc[select.index][var].quantile(0.75)) 
+    
+            data_obs.append(vals_obs.loc[select.index][var].mean()) 
+            data_obs_025.append(vals_obs.loc[select.index][var].quantile(0.25)) 
+            data_obs_075.append(vals_obs.loc[select.index][var].quantile(0.75)) 
+    
+            data_ini.append(ini.loc[select.index][var].mean()) 
+            data_ini_025.append(ini.loc[select.index][var].quantile(0.25)) 
+            data_ini_075.append(ini.loc[select.index][var].quantile(0.75)) 
+        else:
+            lnts.append(0)
+            data.append(np.nan)
+            data_025.append(np.nan)
+            data_075.append(np.nan)
+            data_fc.append(np.nan)
+            data_fc_025.append(np.nan)
+            data_fc_075.append(np.nan)
+    
+            data_obs.append(np.nan)
+            data_obs_025.append(np.nan)
+            data_obs_075.append(np.nan)
+    
+    
+            data_ini.append(np.nan)
+            data_ini_025.append(np.nan)
+            data_ini_075.append(np.nan)
+    
+    data = np.array(data)
+    data_025 = np.array(data_025)
+    data_075 = np.array(data_075)
+    
+    data_fc = np.array(data_fc)
+    data_fc_025 = np.array(data_fc_025)
+    data_fc_075 = np.array(data_fc_075)
+    
+    
+    data_obs = np.array(data_obs)
+    data_obs_025 = np.array(data_obs_025)
+    data_obs_075 = np.array(data_obs_075)
+    
+    data_ini = np.array(data_ini)
+    data_ini_025 = np.array(data_ini_025)
+    data_ini_075 = np.array(data_ini_075)
+
+    if var == 'q':
+        data = data*1000.
+        data_025 = data_025*1000.
+        data_075 = data_075*1000.
+
+        data_fc = data_fc*1000.
+        data_fc_025 = data_fc_025*1000.
+        data_fc_075 = data_fc_075*1000.
+    
+        data_obs = data_obs*1000.
+        data_obs_025 = data_obs_025*1000.
+        data_obs_075 = data_obs_075*1000.
+
+        data_ini = data_ini*1000.
+        data_ini_025 = data_ini_025*1000.
+        data_ini_075 = data_ini_075*1000.
+    
+
+    
+
+    data_left = np.zeros_like(data)
+    data_right = np.zeros_like(data)
+    select = (data >= data_ini)
+    data_left[select] = data[select]
+    data_right[select] = data_ini[select]
+    data_left[~select] = data_ini[~select]
+    data_right[~select] = data[~select]
+
+    data_diff_left = np.zeros_like(data)
+    data_diff_right = np.zeros_like(data)
+    select = (data_fc >= data)
+    data_diff_left[select] = data[select]
+    data_diff_right[select] = data_fc[select]
+    data_diff_left[~select] = data_fc[~select]
+    data_diff_right[~select] = data[~select]
+
+
+    
+    
+    #bar = ax.barh(lats, data,blat[1:] - blat[:-1] , align="center", xerr=error)
+    
+    
+    erb = ax.errorbar( data_ini, lats+2.*dlat/8.,xerr=[data_ini-data_ini_025,data_ini_075-data_ini], fmt='s', color='darkgrey',mfc='white', ms=3, mew=1)
+    erb = ax.errorbar( data_obs, lats+2.*dlat/8.,xerr=[data_obs-data_obs_025,data_obs_075-data_obs], fmt='s', color='darkgrey',mfc='black', ms=3, mew=1)
+    erb = ax.errorbar( data,    lats-1.*dlat/8,xerr=[data-data_025,data_075-data], fmt='s', color='black',mfc='black', ms=3, mew=1)
+    er2 = ax.errorbar( data_fc, lats-2*dlat/8.,xerr=[data_fc-data_fc_025,data_fc_075-data_fc], fmt='s', color='blue',mfc='blue', ms=3, mew=1)
+    
+    ba2 = ax.barh(lats, data_diff_right - data_diff_left  ,(blat[1:] -
+                                                            blat[:-1])*0.85 ,
+                  align="center", left=data_diff_left,color='red',
+                  edgecolor='lightgrey',linewidth=2.)
+    bar = ax.barh(lats, data_right - data_left ,(blat[1:] - blat[:-1])*0.85 ,
+                  align="center", left=data_left, color='none',edgecolor='black',linewidth=2.)
+    # ax.set_yticks(blat)
+    # labels = [ w.get_text() for w in ax.get_yticklabels()]
+    # ax.set_yticklabels(labels)
+    # ax.set_yticks(blat)
+    ax.set_yticks([["test",-30.]])
+    if ivar == 0:
+        plt.legend([r"morning observations",\
+                    r"afternoon observations",\
+                    r"afternoon control",\
+                    r"afternoon $\it{wet}$ "])
+    
+    #er2 = ax.errorbar( data_fc, lats,xerr=[data_fc-data_fc_025,data_fc_075-data_fc], fmt='s', mfc='blue', ms=3, mew=1)
+    
+    
+    
+    # plot = ax.plot(x, data)
+    ax.set_yticks(lats)
+    ax.set_xlim(xlims[ivar])
+    ax.set_ylim((-65.,65.))
+    #ax.set_yticklabels(('wt', 'N23PP', 'N23PP/PEKN', 'PEKN', 'N23PP/PEKN/L28F'))
+    #ax.set_title(r"Everything in the document can use m$\alpha$th language", y=1.05)
+    ax.set_title(labels[ivar],fontsize=17.)
+    ax.set_xlabel("["+units[ivar]+"]", labelpad=10,fontsize=15.)
+    if ivar == 0:
+        ax.set_ylabel("Latitude [°]",labelpad=10,fontsize=15.)
+    ax.yaxis.set_ticks_position('left')
+    ax.xaxis.set_ticks_position('bottom')
+    ax.tick_params(labelsize=15.)
+fig.tight_layout()
+fig.subplots_adjust(left=0.10,bottom=0.13,right=0.98,top=0.94,wspace=0.24,hspace=0.20)
+fig.savefig('test.png',dpi=200)
+fig.show()
+
+
diff --git a/class4gl/interface_functions.py b/class4gl/interface_functions.py
new file mode 100644
index 0000000..fbc335a
--- /dev/null
+++ b/class4gl/interface_functions.py
@@ -0,0 +1,650 @@
+import pandas as pd
+import numpy as np
+import datetime as dt
+import os
+import xarray as xr
+import sys
+from contextlib import suppress
+from time import sleep
+
+from tempfile import gettempdir
+
+
+from class4gl import class4gl_input, data_global,class4gl,units
+from interface_functions import *
+#from data_soundings import wyoming
+import yaml
+from yaml import CLoader
+import glob
+import pandas as pd
+import json
+import io
+import subprocess
+import pytz
+from scipy.stats import mstats
+
+from matplotlib.colors import LinearSegmentedColormap
+
+def which(program):
+    import os
+    def is_exe(fpath):
+        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+    fpath, fname = os.path.split(program)
+    if fpath:
+        if is_exe(program):
+            return program
+    else:
+        for path in os.environ["PATH"].split(os.pathsep):
+            exe_file = os.path.join(path, program)
+            if is_exe(exe_file):
+                return exe_file
+
+    return None
+
+
+
+TEMPDIR = gettempdir() #.replace('[',"").replace(']',"")
+#TEMPDIR = '/tmp/'
+
+class records_iterator(object):
+    def __init__(self,records):
+            
+        self.records = records
+        self.ix = -1 
+        
+    def __iter__(self):
+        return self
+
+    def __next__(self,jump=1):
+        self.ix = (self.ix+jump) 
+        if self.ix >= len(self.records.index):
+            raise StopIteration
+        if self.ix < 0:
+            raise StopIteration
+
+        return self.records.index[self.ix], self.records.iloc[self.ix]
+    def __prev__(self):
+        return self.__next__(self,jump=-1)
+
+
+#'_afternoon.yaml'
+def get_record_yaml(yaml_file,index_start,index_end,mode='model_output'):
+    filename = yaml_file.name
+    #filename = path_yaml+'/'+format(current_station.name,'05d')+suffix
+    #yaml_file = open(filename)
+    shortfn = filename.split('/')[-1]
+
+    #print('going to next observation',filename)
+    yaml_file.seek(index_start)
+
+    print('index_start',index_start)
+    print('index_end',index_end)
+    buf =  yaml_file.read(index_end- index_start).replace('inf','9e19').replace('nan','9e19').replace('---','')
+
+    # # Ruby way -> fast
+    # os.system('mkdir -p '+TEMPDIR)
+    # filebuffer = open(TEMPDIR+'/'+shortfn+'.buffer.yaml.'+str(index_start),'w')
+    # filebuffer.write(buf)
+    # filebuffer.close()
+    # # print("HHHEEELOOOO",filename+'.buffer.yaml'+str(index_start))
+    # 
+    # if which('ruby') is None:
+    #     raise RuntimeError ('ruby is not found. Aborting...')
+    # command = 'ruby -rjson -ryaml -e "'+"puts YAML.load_file('"+TEMPDIR+'/'+shortfn+".buffer.yaml."+str(index_start)+"').to_json"+'" > '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start)+' '
+
+    # #command = '/apps/gent/CO7/sandybridge/software/Ruby/2.4.2-foss-2017b/bin/ruby -rjson -ryaml -e "'+"puts YAML.load(ARGF.read()).to_json"+'"'
+    # print(command)
+    # os.system(command)
+    # jsonstream = open(TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start))
+    # record_dict = json.load(jsonstream)
+    # jsonstream.close()
+    # os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.yaml.'+str(index_start))
+
+    record_dict = yaml.load(buf,Loader=CLoader)
+
+
+    if mode =='model_output':
+        modelout = class4gl()
+        modelout.load_yaml_dict(record_dict)
+
+        # # needed in case of Ruby
+        # os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start))
+        print('hello')
+
+        return modelout
+    elif mode == 'model_input':
+
+ 
+        # datetimes are incorrectly converted to strings. We need to convert them
+        # again to datetimes
+        for key,value in record_dict['pars'].items():
+            # # needed in case of ruby
+            # we don't want the key with columns that have none values
+            # if value is not None: 
+            #     if key in ['lSunrise','lSunset','datetime','ldatetime','ldatetime_daylight','datetime_daylight',]:#(type(value) == str):
+            #    # elif (type(value) == str):
+            #         record_dict['pars'][key] = dt.datetime.strptime(value,"%Y-%m-%d %H:%M:%S %z")
+
+            if (value == 0.9e19) or (value == '.9e19'):
+                record_dict['pars'][key] = np.nan
+        for key in record_dict.keys():
+            #print(key)
+            if key in ['air_ap','air_balloon',]:
+                #NNprint('check')
+                for datakey,datavalue in record_dict[key].items():
+                    record_dict[key][datakey] = [ np.nan if (x =='.9e19') else x for x in record_dict[key][datakey]]
+
+
+        c4gli = class4gl_input()
+        #print(c4gli.logger,'hello')
+        c4gli.load_yaml_dict(record_dict)
+
+        # # needed in case of ruby
+        # os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start))
+        return c4gli
+    else:
+        print('Warning. Mode '+mode+' not recorgnized. Returning None')
+        return None
+
+
+
+
+
+
+        # self.frames['stats']['records_current_station_index'] = \
+        #     (self.frames['stats']['records_all_stations_index'].get_level_values('STNID')\
+        #      == \
+        #      self.frames['stats']['current_station'].name)
+
+        # # create the value table of the records of the current station
+        # tab_suffixes = \
+        #         ['_mod','_obs','_obs_afternoon','_mod_stats','_obs_afternoon_stats','_ini_pct']
+        # for tab_suffix in tab_suffixes:
+        #     self.frames['stats']['records_current_station'+tab_suffix] = \
+        #         self.frames['stats']['records_all_stations'+tab_suffix].iloc[self.frames['stats']['records_current_station_index']]
+
+
+# class records_selection(object):
+#     def __init__
+
+# class records(object):
+#     def __init__(self,stations,path_obs,path_mod):
+#         self.stations = stations
+#         self.path_obs = path_obs
+#         self.path_mod = path_mod
+# 
+#         self.ini =       self.get_records(self.path_mod,'ini')
+#         self.mod =       self.get_records(self.path_mod,'mod')
+#         #self.morning =   self.get_records(self.path_obs,'morning')
+#         self.afternoon = self.get_records(self.path_obs,'afternoon')
+# 
+#         
+#         self.afternoon.index = self.afternoon.ldatetime.dt.date
+#         self.afternoon = self.afternoon.loc[records_ini.ldatetime.dt.date]
+# 
+#         self.index = self.ini.index
+#         self.mod.index = self.index
+#         self.afternoon.index = self.index
+# 
+# 
+#         #self.records_iterator = records_current_station_mod.iterrows()
+
+
+
+
+class stations(object):
+    def __init__(self,path,suffix='ini',refetch_stations=True):
+
+        self.path = path
+
+        self.file = self.path+'/stations_list.csv'
+        if (os.path.isfile(self.file)) and (not refetch_stations):
+            self.table = pd.read_csv(self.file)
+        else:
+            self.table = self.get_stations(suffix=suffix)
+            self.table.to_csv(self.file)
+        
+        print(self.table.columns)
+        self.table = self.table.set_index('STNID')
+
+    def get_stations(self,suffix):
+        print(suffix)
+        stations_list_files = glob.glob(self.path+'/?????_*_'+suffix+'.yaml')
+        if len(stations_list_files) == 0:
+            stations_list_files = glob.glob(self.path+'/?????_'+suffix+'.yaml')
+        else:
+            # this weird section retreives the first file of every station
+            stations_list_files_1 = [station_file[:len(self.path+'/?????')] for \
+                                   station_file in stations_list_files]
+            stations_list_files_2 = [station_file[len(self.path+'/?????'):] for \
+                                   station_file in stations_list_files]
+            #print(stations_list_files_1)
+            stations_list_files_new = []
+            stations_list_files_skip = []
+            for istat,stations_file_1 in  enumerate(stations_list_files_1):
+                if stations_file_1 not in stations_list_files_skip:
+                    stations_list_files_skip.append(stations_file_1)
+                    stations_list_files_new.append(stations_file_1+stations_list_files_2[istat])
+            stations_list_files = stations_list_files_new
+            
+        stations_list_files.sort()
+
+        if len(stations_list_files) == 0:
+            raise ValueError('no stations found that match "'+self.path+'/?????[_0]_'+suffix+'.yaml'+'"')
+        stations_list = []
+        for stations_list_file in stations_list_files:
+            thisfile = open(stations_list_file,'r')
+            yamlgen = yaml.load_all(thisfile,Loader=CLoader)
+            try:
+                first_record  = yamlgen.__next__()
+            except:
+                first_record = None
+            if first_record is not None:
+                stations_list.append({})
+                for column in ['STNID','latitude','longitude']:
+                    #print(first_record['pars'].keys())
+                    stations_list[-1][column] = first_record['pars'][column]
+                stations_list[-1]['filename'] = os.path.split(stations_list_file)[1]
+            yamlgen.close()
+            thisfile.close()
+    
+        #print(stations_list)
+        return pd.DataFrame(stations_list)
+
+class stations_iterator(object):
+    def __init__(self,stations):
+        self.stations = stations
+        self.ix = -1 
+        print('self.ix',self.ix)
+        print(stations.table)
+    def __iter__(self):
+        return self
+    def __next__(self,jump=1):
+        print('next jump',jump)
+        print('self.ix',self.ix)
+        if ((self.ix+jump >= len(self.stations.table.index)) or (self.ix+jump < 0 )):
+            raise StopIteration
+        self.ix = (self.ix+jump) 
+        print('self.ix',self.ix)
+        self.ix = np.mod(self.ix,len(self.stations.table)) 
+        print('self.ix',self.ix)
+        return self.stations.table.index[self.ix], self.stations.table.iloc[self.ix]
+    def set_row(self,row):
+        self.ix = row
+        return self.stations.table.index[self.ix], self.stations.table.iloc[self.ix]
+    def set_STNID(self,STNID):
+        self.ix = np.where((self.stations.table.index == STNID))[0][0]
+        #print(self.ix)
+        #print( self.stations.table.index[self.ix], self.stations.table.iloc[self.ix])
+        return self.stations.table.index[self.ix], self.stations.table.iloc[self.ix]
+
+    def __prev__(self):
+        return self.__next__(self,jump=-1)
+    def close():
+        del(self.ix)
+
+class records_iterator(object):
+    def __init__(self,records):
+            
+        self.records = records
+        self.ix = -1 
+        
+    def __iter__(self):
+        return self
+
+    def __next__(self,jump=1):
+        self.ix = (self.ix+jump) 
+        if self.ix >= len(self.records.index):
+            raise StopIteration
+        self.ix = np.mod(self.ix,len(self.records))
+        return self.records.index[self.ix], self.records.iloc[self.ix]
+    def __prev__(self):
+        return self.__next__(self,jump=-1)
+
+
+# #'_afternoon.yaml'
+# def get_record_yaml(yaml_file,index_start,index_end):
+#     filename = yaml_file.name
+#     #filename = path_yaml+'/'+format(current_station.name,'05d')+suffix
+#     #yaml_file = open(filename)
+# 
+#     #print('going to next observation',filename)
+#     yaml_file.seek(index_start)
+# 
+#     buf =  yaml_file.read(index_end- index_start).replace('inf','9e19').replace('nan','9e19').replace('---','')
+# 
+#     filebuffer = open(filename+'.buffer.yaml.'+str(index_start),'w')
+#     filebuffer.write(buf)
+#     filebuffer.close()
+#     # print("HHHEEELOOOO",filename+'.buffer.yaml'+str(index_start))
+#     
+#     command = '/apps/gent/CO7/sandybridge/software/Ruby/2.4.2-foss-2017b/bin/ruby -rjson -ryaml -e "'+"puts YAML.load_file('"+filename+".buffer.yaml."+str(index_start)+"').to_json"+'" > '+filename+'.buffer.json.'+str(index_start)+' '
+# 
+#     #command = '/apps/gent/CO7/sandybridge/software/Ruby/2.4.2-foss-2017b/bin/ruby -rjson -ryaml -e "'+"puts YAML.load(ARGF.read()).to_json"+'"'
+#     print(command)
+#     os.system(command)
+#     jsonstream = open(filename+'.buffer.json.'+str(index_start))
+#     record_dict = json.load(jsonstream)
+#     jsonstream.close()
+#     os.system('rm '+filename+'.buffer.yaml.'+str(index_start))
+#  
+#     # datetimes are incorrectly converted to strings. We need to convert them
+#     # again to datetimes
+#     for key,value in record_dict['pars'].items():
+#         # we don't want the key with columns that have none values
+#         if value is not None: 
+#             if key in ['lSunrise','lSunset','datetime','ldatetime','ldatetime_daylight','ldatetime_daylight','datetime_daylight','datetime_daylight']:#(type(value) == str):
+#            # elif (type(value) == str):
+#                 record_dict['pars'][key] = dt.datetime.strptime(value,"%Y-%m-%d %H:%M:%S %z")
+#                 
+#                 # Workaround. Unfortunately, Ruby puts it in local time of the computer. Turn it back to UTC (note that UTC means actually local time)!!!
+#                 record_dict['pars'][key] = record_dict['pars'][key].astimezone(pytz.UTC)
+# 
+#         if (value == 0.9e19) or (value == '.9e19'):
+#             record_dict['pars'][key] = np.nan
+#     for key in record_dict.keys():
+#         print(key)
+#         if key in ['air_ap','air_balloon',]:
+#             print('check')
+#             for datakey,datavalue in record_dict[key].items():
+#                 record_dict[key][datakey] = [ np.nan if (x =='.9e19') else x for x in record_dict[key][datakey]]
+# 
+#     #os.system('rm '+filename+'.buffer.json.'+str(index_start))
+# 
+#     c4gli = class4gl_input()
+#     c4gli.load_yaml_dict(record_dict)
+#     return c4gli
+
+
+
+
+
+
+        # self.frames['stats']['records_current_station_index'] = \
+        #     (self.frames['stats']['records_all_stations_index'].get_level_values('STNID')\
+        #      == \
+        #      self.frames['stats']['current_station'].name)
+
+        # # create the value table of the records of the current station
+        # tab_suffixes = \
+        #         ['_mod','_obs','_obs_afternoon','_mod_stats','_obs_afternoon_stats','_ini_pct']
+        # for tab_suffix in tab_suffixes:
+        #     self.frames['stats']['records_current_station'+tab_suffix] = \
+        #         self.frames['stats']['records_all_stations'+tab_suffix].iloc[self.frames['stats']['records_current_station_index']]
+
+
+# class records_selection(object):
+#     def __init__
+
+# class records(object):
+#     def __init__(self,stations,path_obs,path_mod):
+#         self.stations = stations
+#         self.path_obs = path_obs
+#         self.path_mod = path_mod
+# 
+#         self.ini =       self.get_records(self.path_mod,'ini')
+#         self.mod =       self.get_records(self.path_mod,'mod')
+#         #self.morning =   self.get_records(self.path_obs,'morning')
+#         self.afternoon = self.get_records(self.path_obs,'afternoon')
+# 
+#         
+#         self.afternoon.index = self.afternoon.ldatetime.dt.date
+#         self.afternoon = self.afternoon.loc[records_ini.ldatetime.dt.date]
+# 
+#         self.index = self.ini.index
+#         self.mod.index = self.index
+#         self.afternoon.index = self.index
+# 
+# 
+#         #self.records_iterator = records_current_station_mod.iterrows()
+
+
+
+def get_records(stations,path_yaml,getchunk='all',subset='morning',refetch_records=False):
+    print(stations)
+    records = pd.DataFrame()
+    for STNID,station in stations.iterrows():
+        dictfnchunks = []
+        pklchunks = []
+        if getchunk is 'all':
+
+            # we try the old single-chunk filename format first (usually for
+            # original profile pairs)
+            fn = format(STNID,'05d')+'_'+subset+'.yaml'
+            if os.path.isfile(path_yaml+'/'+fn):
+                chunk = 0
+                dictfnchunks.append(dict(fn=fn,chunk=chunk))
+                pklchunks.append(fn.replace('.yaml','.pkl'))
+
+            # otherwise, we use the new multi-chunk filename format
+            else:
+                chunk = 0
+                end_of_chunks = False
+                station_list_files = glob.glob(path_yaml+'/'+format(STNID,'05d')+'_*_'+subset+'.yaml')
+                station_list_files.sort()
+                chunks = []
+                for station_path_file in station_list_files:
+                    fn = station_path_file.split('/')[-1]
+                    chunks.append(int(fn.split('_')[1]))
+
+                # sort according to chunk number
+                chunks.sort()
+                for chunk in chunks:
+                    fn = format(STNID,'05d')+'_'+str(chunk)+'_'+subset+'.yaml'
+                    dictfnchunks.append(dict(fn=fn,chunk=chunk))
+                    pklchunks.append(fn.replace('.yaml','.pkl'))
+
+                # while not end_of_chunks:
+                #     fn = format(STNID,'05d')+'_'+str(chunk)+'_'+subset+'.yaml'
+                #     if os.path.isfile(path_yaml+'/'+fn):
+                #         dictfnchunks.append(dict(fn=fn,chunk=chunk))
+                #     else:
+                #         end_of_chunks = True
+                #     chunk += 1
+
+            # globyamlfilenames = path_yaml+'/'+format(STNID,'05d')+'*_'+subset+'.yaml'
+            # yamlfilenames = glob.glob(globyamlfilenames)
+            # yamlfilenames.sort()
+        else:
+            fn = format(STNID,'05d')+'_'+str(getchunk)+'_'+subset+'.yaml'
+            dictfnchunks.append(dict(fn=fn,chunk=getchunk))
+
+        if (len(dictfnchunks) > 0):
+            load_from_unified_pkl = False    
+            pklfilename_unified = format(STNID,'05d')+'_'+subset+'.pkl'
+            if (getchunk is 'all') and (os.path.isfile(path_yaml+'/'+pklfilename_unified)):
+                load_from_unified_pkl = True
+                for dictfnchunk in dictfnchunks:
+                    yamlfilename = dictfnchunk['fn']
+                    chunk = dictfnchunk['chunk']
+                    pklfilename = yamlfilename.replace('.yaml','.pkl')
+
+
+                    if \
+                       (pklfilename_unified in pklchunks) or \
+                       (not os.path.isfile(path_yaml+'/'+pklfilename)) or \
+                       (os.path.getmtime(path_yaml+'/'+yamlfilename) > os.path.getmtime(path_yaml+'/'+pklfilename_unified)) or\
+                       (os.path.getmtime(path_yaml+'/'+pklfilename) > os.path.getmtime(path_yaml+'/'+pklfilename_unified)):
+                        load_from_unified_pkl = False
+
+            if load_from_unified_pkl:
+                pklfilename_unified = format(STNID,'05d')+'_'+subset+'.pkl'
+                print('reading unified table file ('+path_yaml+'/'+pklfilename_unified+') for station '\
+                              +str(STNID))
+
+                records_station = pd.read_pickle(path_yaml+'/'+pklfilename_unified)
+            else:
+                records_station = pd.DataFrame()
+                for dictfnchunk in dictfnchunks:
+                    yamlfilename = dictfnchunk['fn']
+                    chunk = dictfnchunk['chunk']
+
+                    #pklfilename = path_yaml+'/'+format(STNID,'05d')+'_'+subset+'.pkl'
+                    pklfilename = yamlfilename.replace('.yaml','.pkl')
+
+                    #print(yamlfilename+": "+str(os.path.getmtime(yamlfilename)))
+                    #print(pklfilename+": "+str(os.path.getmtime(pklfilename)))
+                    generate_pkl = False
+                    if not os.path.isfile(path_yaml+'/'+pklfilename): 
+                        print('pkl file does not exist. I generate "'+\
+                              path_yaml+'/'+pklfilename+'" from "'+path_yaml+'/'+yamlfilename+'"...')
+                        generate_pkl = True
+                    elif refetch_records:
+                        print('refetch_records flag is True. I regenerate "'+\
+                              path_yaml+'/'+pklfilename+'" from "'+path_yaml+'/'+yamlfilename+'"...')
+                        generate_pkl = True
+                    elif not (os.path.getmtime(path_yaml+'/'+yamlfilename) <  \
+                        os.path.getmtime(path_yaml+'/'+pklfilename)):
+                        print('pkl file older than yaml file, so I regenerate "'+\
+                              path_yaml+'/'+pklfilename+'" from "'+path_yaml+'/'+yamlfilename+'"...')
+                        generate_pkl = True
+                    if not generate_pkl:
+                        records_station_chunk = pd.read_pickle(path_yaml+'/'+pklfilename)
+                        records_station = pd.concat([records_station,records_station_chunk],sort=True)
+                       # irecord = 0
+                    else:
+                        with open(path_yaml+'/'+yamlfilename) as yaml_file:
+
+                            dictout = {}
+
+                            #initialization (go to file position just after "---")
+                            next_record_found = False
+                            end_of_file = False
+                            while (not next_record_found) and (not end_of_file):
+                                linebuffer = yaml_file.readline()
+                                next_record_found = (linebuffer == '---\n')
+                                end_of_file = (linebuffer == '')
+                            next_tell = yaml_file.tell()
+                            
+                            # loop over different yaml profile records
+                            while not end_of_file:
+
+                                print(' next record:',next_tell)
+                                current_tell = next_tell
+                                next_record_found = False
+                                yaml_file.seek(current_tell)
+
+                                # # needed for Ruby
+                                # os.system('mkdir -p '+TEMPDIR)
+                                # filebuffer = open(TEMPDIR+'/'+yamlfilename+'.buffer.yaml.'+str(current_tell),'w')
+
+                                linebuffer = ''
+                                stringbuffer = ''
+                                while ( (not next_record_found) and (not end_of_file)):
+                                    # # needed for Ruby
+                                    # filebuffer.write(linebuffer.replace('inf','0').replace('nan','0'))
+                                    stringbuffer += linebuffer 
+                                    linebuffer = yaml_file.readline()
+                                    next_record_found = (linebuffer == '---\n')
+                                    end_of_file = (linebuffer == '')
+                                # # needed for Ruby
+                                # filebuffer.close()
+                                
+                                next_tell = yaml_file.tell()
+                                index_start = current_tell
+                                index_end = next_tell
+
+                                # start direct yaml way -> slow
+                                record = yaml.load(stringbuffer,Loader=CLoader)
+                                # end direct way
+
+                                # # # start json ruby way -> much faster 
+                                # if which('ruby') is None:
+                                #     raise RuntimeError ('ruby is not found. Aborting...')
+                                # #if ((irecord >= start) and (np.mod(irecord - start,2) == 0.) :
+                                # command = 'ruby -rjson -ryaml -e "'+"puts YAML.load_file('"+TEMPDIR+'/'+yamlfilename+".buffer.yaml."+str(current_tell)+"').to_json"+'" > '+TEMPDIR+'/'+yamlfilename+'.buffer.json.'+str(current_tell)+' ' 
+                                # print(command)
+                                # 
+                                # os.system(command)
+                                # #jsonoutput = subprocess.check_output(command,shell=True) 
+                                # #print(jsonoutput)
+                                # #jsonstream = io.StringIO(jsonoutput)
+                                # jsonstream = open(TEMPDIR+'/'+yamlfilename+'.buffer.json.'+str(current_tell))
+                                # record = json.load(jsonstream)
+                                # # end json ruby way
+
+
+                                dictouttemp = {}
+                                for key,value in record['pars'].items():
+                                    # we don't want the key with columns that have none values
+                                    if value is not None: 
+                                       regular_numeric_types =[ type(x) for x in[0,False,0.0]]
+                                       if (type(value) in regular_numeric_types):
+                                            dictouttemp[key] = value
+                                       elif key in ['lSunrise','lSunset','datetime','ldatetime','datetime_daylight','datetime_daylight','ldatetime_daylight','ldatetime_daylight']:#(type(value) == str):
+                                           dictouttemp[key] = value
+                                           # # needed for Ruby
+                                           # dictouttemp[key] = dt.datetime.strptime(value,"%Y-%m-%d %H:%M:%S %z")
+
+                                           # # Workaround. Unfortunately, Ruby puts it in local time of the computer. Turn it back to UTC (note that UTC means actually local time)!!!
+                                           # dictouttemp[key] = dictouttemp[key].astimezone(pytz.UTC)
+                                recordindex = record['index']
+                                dictouttemp['chunk'] = chunk
+                                dictouttemp['index_start'] = index_start
+                                dictouttemp['index_end'] = index_end
+                                # os.system('rm '+TEMPDIR+'/'+yamlfilename+'.buffer.json.'+str(current_tell))
+                                for key,value in dictouttemp.items():
+                                    if key not in dictout.keys():
+                                        dictout[key] = {}
+                                    dictout[key][(STNID,chunk,recordindex)] = dictouttemp[key]
+                                print(' obs record registered')
+                                # # needed for Ruby
+                                # jsonstream.close()
+                                #  os.system('rm '+TEMPDIR+'/'+yamlfilename+'.buffer.yaml.'+str(current_tell))
+                            records_station_chunk = pd.DataFrame.from_dict(dictout)
+                            if len(records_station_chunk) > 0:
+                                records_station_chunk.index.set_names(('STNID','chunk','index'),inplace=True)
+                                print('writing table file ('+path_yaml+'/'+pklfilename+') for station '\
+                                      +str(STNID)+', chunk number '+str(chunk))
+                                records_station_chunk.to_pickle(path_yaml+'/'+pklfilename)
+                                records_station = pd.concat([records_station,records_station_chunk])
+                            else:
+                                print('Warning. No records found in ',yaml_file)
+                        # else:
+                        #     os.system('rm '+pklfilename)
+                if (getchunk == 'all') and (pklfilename_unified not in pklchunks):
+                    pklfilename_unified = format(STNID,'05d')+'_'+subset+'.pkl'
+                    print('writing unified table file ('+path_yaml+'/'+pklfilename_unified+') for station '\
+                                  +str(STNID))
+                    records_station.to_pickle(path_yaml+'/'+pklfilename_unified)
+
+            records = pd.concat([records,records_station],sort=True)
+    return records
+
+def stdrel(mod,obs,columns):
+    stdrel = pd.DataFrame(columns = columns)
+    for column in columns:
+        stdrel[column] = \
+                (mod.groupby('STNID')[column].transform('mean') -
+                 obs.groupby('STNID')[column].transform('mean')) /\
+                obs.groupby('STNID')[column].transform('std') + \
+                (mod[column] -
+                 mod.groupby('STNID')[column].transform('mean')) /\
+                obs.groupby('STNID')[column].transform('std') 
+    return stdrel
+
+def pct(obs,columns):
+    pct = pd.DataFrame(columns=columns)
+    for column in columns:
+        #print(column)
+        pct[column] = ""
+        pct[column] = obs[column].rank(pct=True)
+    return pct
+
+def tendencies(mod_afternoon,obs_afternoon,obs_morning,keys):
+    stats = pd.DataFrame()
+    for key in keys: 
+        stats['d'+key+'dt'] = ""
+        stats['d'+key+'dt'] = (mod_afternoon[key] - obs_morning[key])/ \
+                              (obs_afternoon.ldatetime - \
+                               obs_morning.ldatetime).dt.seconds*3600.
+    return stats
+def tendencies_rev(mod_afternoon,mod_ini,keys):
+    stats = pd.DataFrame()
+    for key in keys: 
+        stats['d'+key+'dt'] = ""
+        stats['d'+key+'dt'] = (mod_afternoon[key] - mod_ini[key])/ \
+                              (mod_ini.runtime)
+    return stats
+
diff --git a/class4gl/interface_multi.py b/class4gl/interface_multi.py
new file mode 100644
index 0000000..0c5a0dd
--- /dev/null
+++ b/class4gl/interface_multi.py
@@ -0,0 +1,2382 @@
+import pandas as pd
+import numpy as np
+import datetime as dt
+import os
+import xarray as xr
+import sys
+from contextlib import suppress
+from time import sleep
+import copy
+import matplotlib.image as mpimg
+
+
+# sys.path.insert(0, '/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/')
+
+import class4gl
+path_lib = os.path.dirname(class4gl.__file__)
+path_worldmap = path_lib+'/Equirectangular_projection_SW.png'
+img_worldmap= mpimg.imread(path_worldmap)
+
+
+from class4gl import class4gl_input, data_global,class4gl,units
+from interface_functions import *
+# from data_soundings import wyoming
+import yaml
+import glob
+import pandas as pd
+import json
+import io
+import subprocess
+import pytz
+from scipy.stats import mstats
+
+from matplotlib.colors import LinearSegmentedColormap
+cdictpres = {'blue': (\
+                   (0.,    0.,  0.),
+                   (0.25,  0.25, 0.25),
+                   (0.5,  .70, 0.70),
+                   (0.75, 1.0, 1.0),
+                   (1,     1.,  1.),
+                   ),
+       'green': (\
+                   (0. ,   0., 0.0),
+                   (0.25,  0.50, 0.50),
+                   (0.5,  .70, 0.70),
+                   (0.75,  0.50, 0.50),
+                   (1  ,    0,  0.),
+                   ),
+       'red':  (\
+                  (0 ,  1.0, 1.0),
+                  (0.25 ,  1.0, 1.0),
+                   (0.5,  .70, 0.70),
+                  (0.75 , 0.25, 0.25),
+                  (1,    0., 0.),
+                  )}
+
+statsviewcmap = LinearSegmentedColormap('statsviewcmap', cdictpres)
+
+
+os.system('module load Ruby')
+
+class c4gl_interface_soundings(object):
+    def __init__(self,path_exp,path_forcing=None,globaldata=None,refetch_records=False,end_state=True,refetch_stations=True,inputkeys = ['cveg','wg','w2','cc','sp','wwilt','Tsoil','T2','z0m','alpha','LAI',],obs_filter=False,tendencies_revised=False):
+        """ creates an interactive interface for analysing class4gl experiments
+
+        INPUT:
+            path_exp : path of the experiment output
+            path_forcing : path of the original forcing, which is needed to get the end (afternoon) profiles
+            globaldata: global data that is being shown on the map
+            obs_filtering: extra data filter considering observation tendencies
+                           beyond what the model can capture
+            refetch_stations: do we need to build the list of the stations again?
+        OUTPUT:
+            the procedure returns an interface object with interactive plots
+
+        """
+        
+        # set the ground
+        self.globaldata = globaldata
+
+ 
+        self.obs_filter = obs_filter
+        print(self.obs_filter)
+        self.tendencies_revised = tendencies_revised
+        self.path_exp = path_exp
+        self.path_forcing = path_forcing
+        self.end_state = end_state
+        self.exp_files = glob.glob(self.path_exp+'/?????.yaml')
+
+        # # get the list of stations
+        # stationsfile = self.path_exp+'/stations_list.csv'
+        # if (os.path.isfile(stationsfile)) and (not refetch_stations):
+        #     stations = pd.read_csv(stationsfile)
+        # else:
+        #     stations = get_stations(self.path_exp)
+        #     stations.to_csv(stationsfile)
+
+        # stations = stations.set_index('STNID')
+
+        self.frames = {}
+
+        self.frames['stats'] = {}
+        self.frames['worldmap'] = {}
+                
+        self.frames['profiles'] = {}
+        self.frames['profiles'] = {}
+        self.frames['profiles']['DT'] = None
+        self.frames['profiles']['STNID'] = None
+
+        #self.frames['worldmap']['stationsfile'] = stationsfile
+        self.frames['worldmap']['stations'] = stations(self.path_exp, \
+                                                       suffix='ini',\
+                                                       refetch_stations=refetch_stations)
+
+        # Initially, the stats frame inherets the values/iterators of
+        # worldmap
+        for key in self.frames['worldmap'].keys():
+            self.frames['stats'][key] = self.frames['worldmap'][key]
+
+        # get its records and load it into the stats frame
+        self.frames['stats']['records_all_stations_ini'] =\
+                        get_records(self.frames['stats']['stations'].table,\
+                                           self.path_exp,\
+                                           subset='ini',\
+                                           refetch_records=refetch_records
+                                           )
+        print(self.frames['stats']['records_all_stations_ini'])
+        # get its records and load it into the stats frame
+
+
+        self.frames['stats']['records_all_stations_end_mod'] =\
+                        get_records(self.frames['stats']['stations'].table,\
+                                           self.path_exp,\
+                                           subset='end',\
+                                           refetch_records=refetch_records
+                                           )
+
+        if self.path_forcing is not None:
+            # get its records and load it into the stats frame
+            self.frames['stats']['records_all_stations_end_obs'] =\
+                            get_records(self.frames['stats']['stations'].table,\
+                                               self.path_forcing,\
+                                               subset='end',\
+                                               refetch_records=refetch_records
+                                               )
+        if self.end_state:
+            if len(self.frames['stats']['records_all_stations_end_mod']) == 0:
+                raise IOError ('No end state records found. If you want to ignore end states, please use the option end_state = False')
+            self.frames['stats']['records_all_stations_end_mod'].index = \
+                self.frames['stats']['records_all_stations_ini'].index 
+
+        else:
+                self.frames['stats']['records_all_stations_end_mod'] = \
+                self.frames['stats']['records_all_stations_ini']
+
+        
+        if len(self.frames['stats']['records_all_stations_ini']) ==0:
+            raise ValueError('no class records found. Aborting')
+
+        self.frames['stats']['records_all_stations_ini']['dates'] = \
+            self.frames['stats']['records_all_stations_ini']['ldatetime'].dt.date
+
+        if self.path_forcing is not None:
+            self.frames['stats']['records_all_stations_end_obs']['dates'] = \
+                self.frames['stats']['records_all_stations_end_obs']['ldatetime'].dt.date
+
+            self.frames['stats']['records_all_stations_end_obs'].set_index(['STNID','dates'],inplace=True)
+
+
+            ini_index_dates = self.frames['stats']['records_all_stations_ini'].set_index(['STNID','dates']).index
+
+            self.frames['stats']['records_all_stations_end_obs'] = \
+                self.frames['stats']['records_all_stations_end_obs'].loc[ini_index_dates]
+
+            self.frames['stats']['records_all_stations_end_obs'].index = \
+                self.frames['stats']['records_all_stations_ini'].index 
+
+            self.frames['stats']['viewkeys'] = ['h','theta','q']
+            print('Calculating table statistics')
+
+            if self.tendencies_revised:
+                self.frames['stats']['records_all_stations_end_mod_stats'] = \
+                        tendencies_rev(self.frames['stats']['records_all_stations_end_mod'],\
+                                           self.frames['stats']['records_all_stations_ini'],\
+                                           self.frames['stats']['viewkeys']\
+                                  )
+                self.frames['stats']['records_all_stations_end_obs_stats'] = \
+                        tendencies_rev(self.frames['stats']['records_all_stations_end_obs'],\
+                                           self.frames['stats']['records_all_stations_ini'],\
+                                           self.frames['stats']['viewkeys']\
+                                  )
+
+            else:
+                self.frames['stats']['records_all_stations_end_mod_stats'] = \
+                        tendencies(self.frames['stats']['records_all_stations_end_mod'],\
+                                   self.frames['stats']['records_all_stations_end_obs'],\
+                                   self.frames['stats']['records_all_stations_ini'],\
+                                   self.frames['stats']['viewkeys']\
+                                  )
+                self.frames['stats']['records_all_stations_end_obs_stats'] = \
+                        tendencies(self.frames['stats']['records_all_stations_end_obs'],\
+                                   self.frames['stats']['records_all_stations_end_obs'],\
+                                   self.frames['stats']['records_all_stations_ini'],\
+                                   self.frames['stats']['viewkeys']\
+                                  )
+
+        self.frames['stats']['inputkeys'] = inputkeys
+        
+        # self.frames['stats']['inputkeys'] = \
+        #     [ key for key in \
+        #       self.globaldata.datasets.keys() \
+        #       if key in \
+        #       list(self.frames['stats']['records_all_stations_obs'].columns)]
+
+
+        # get units from the class4gl units database
+        self.units = dict(units)
+        # for those that don't have a definition yet, we just ask a question
+        # mark
+        for var in self.frames['stats']['inputkeys']:
+            self.units[var] = '?'
+
+        self.frames['worldmap']['inputkeys'] = self.frames['stats']['inputkeys'] 
+        self.frames['stats']['records_all_stations_ini_pct'] = \
+                  pct(self.frames['stats']['records_all_stations_ini'], \
+                      columns = self.frames['stats']['inputkeys'])
+
+        #     pd.DataFrame(columns = self.frames['stats']['viewkeys'])
+        # for ikey,key in enumerate(self.frames['stats']['viewkeys']):
+        #     mod['
+
+        # 
+        # 
+        # \
+        #        self.frames['stats']['records_all_stations_end_mod'], \
+
+
+
+        # self.frames['stats']['records_all_stations_end_mod_stats_stdrel'] = \
+        #        stdrel(mod = self.frames['stats']['records_all_stations_end_mod_stats'], \
+        #               obs = self.frames['stats']['records_all_stations_end_obs_stats'], \
+        #               columns = [ 'd'+key+'dt' for key in \
+        #                           self.frames['stats']['viewkeys']], \
+        #              )
+
+        # self.frames['stats']['records_all_stations_end_obs_stats_stdrel'] = \
+        #        stdrel(mod = self.frames['stats']['records_all_stations_ini'], \
+        #               obs = self.frames['stats']['records_all_stations_ini'], \
+        #               columns = self.frames['stats']['viewkeys'], \
+        #              )
+
+        
+
+        if self.path_forcing is not None:
+            print('filtering pathological data')
+            indextype = self.frames['stats']['records_all_stations_end_mod_stats'].index.names
+            # some observational sounding still seem problematic, which needs to be
+            # investigated. In the meantime, we filter them
+
+            print('hello',self.obs_filter)
+            print ((self.path_forcing is not None) and (self.obs_filter))
+            if ((self.path_forcing is not None) and (self.obs_filter)) is True:
+                print('hallohallo')
+            if ((self.path_forcing is not None) and (self.obs_filter)) is True:
+                print('exclude exceptional observations')
+                print('exclude unrealistic model output -> should be investigated!')
+                valid = (\
+                        (self.frames['stats']['records_all_stations_end_mod'].theta >  273.) & 
+                        (self.frames['stats']['records_all_stations_end_obs'].theta >  273.) & 
+                       #(self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt >  0.25) & 
+                        (self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt >  0.) & 
+                       #(self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt >  0.25000) & 
+                       #(self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt <  1.8000) & 
+                       # (self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt <  2.0000) & 
+                       #(self.frames['stats']['records_all_stations_end_mod_stats'].dhdt >  50.0000) & 
+                        (self.frames['stats']['records_all_stations_end_obs_stats'].dhdt >  0.0000) & 
+                        # (self.frames['stats']['records_all_stations_end_mod_stats'].dhdt <  400.) & 
+                        #  (self.frames['stats']['records_all_stations_end_obs_stats'].dhdt <  350.) & 
+                        #   (self.frames['stats']['records_all_stations_end_obs_stats'].dqdt <  .0003) & 
+                        #   (self.frames['stats']['records_all_stations_end_obs_stats'].dqdt >  -.0006) & 
+                        #   (self.frames['stats']['records_all_stations_end_obs_stats'].dqdt <  .0003) & 
+
+                         # filter 'extreme' model output -> should be investigated!
+                       #    (self.frames['stats']['records_all_stations_end_mod_stats'].dqdt <  .0006) & 
+                       #    (self.frames['stats']['records_all_stations_end_mod_stats'].dqdt >  -.0006) & 
+                          (self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt >  .0) & 
+                       #  (self.frames['stats']['records_all_stations_end_mod_stats'].dhdt >  40.) & 
+                          (self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt >  .0) & 
+                          (self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt <  2.3) & 
+                         # (self.frames['stats']['records_all_stations_end_mod_stats'].dqdt <  .0003) & 
+                         # (self.frames['stats']['records_all_stations_ini'].KGC != 'Cwb') & 
+                         # (self.frames['stats']['records_all_stations_ini'].KGC != 'Dfc') & 
+                         ~np.isnan(self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt) & 
+                         ~np.isnan(self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt))
+# 
+# 
+# 
+# 
+#                     #   (self.frames['stats']['records_all_stations_ini'].lat >-2.  ) & 
+#                          (self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt >  0.00) & 
+#                         (self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt >  0.25000) & 
+#                         (self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt <  3.0000) & 
+#                          (self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt <  1.8000) & 
+#                          #(self.frames['stats']['records_all_stations_end_mod_stats'].dhdt >  50.0000) & 
+#                          (self.frames['stats']['records_all_stations_end_obs_stats'].dhdt >  40.0000) & 
+#                          #(self.frames['stats']['records_all_stations_end_mod_stats'].dhdt <  350.) & 
+#                          (self.frames['stats']['records_all_stations_end_obs_stats'].dhdt <  450.) & 
+#                        (self.frames['stats']['records_all_stations_end_obs_stats'].dqdt >  -0.0006) & 
+#                        (self.frames['stats']['records_all_stations_end_obs_stats'].dqdt <  0.0003) & 
+#                      #   ((self.frames['stats']['records_all_stations_ini'].ldatetime-
+#                      #     self.frames['stats']['records_all_stations_ini'].lSunset).total_seconds()
+#                      #    <= -2.*3600.) & 
+#                          # filter 'extreme' model output -> should be investigated!
+#                         (self.frames['stats']['records_all_stations_end_mod_stats'].dqdt <  .0005) & 
+#                         (self.frames['stats']['records_all_stations_end_mod_stats'].dqdt >  -.0006) & 
+#                          (self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt >  .2) & 
+#                           (self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt <  2.) & 
+#                         # (self.frames['stats']['records_all_stations_end_mod_stats'].dhdt <  400.) & 
+#                          # (self.frames['stats']['records_all_stations_end_mod_stats'].dqdt <  .0003) & 
+#                          # (self.frames['stats']['records_all_stations_ini'].KGC != 'Cwb') & 
+#                          # (self.frames['stats']['records_all_stations_ini'].KGC != 'Dfc') & 
+#                          ~np.isnan(self.frames['stats']['records_all_stations_end_mod_stats'].dthetadt) & 
+#                          ~np.isnan(self.frames['stats']['records_all_stations_end_obs_stats'].dthetadt))
+
+                for key in self.frames['stats'].keys():
+                    if (type(self.frames['stats'][key]) == pd.DataFrame) and \
+                       (self.frames['stats'][key].index.names == indextype):
+                        self.frames['stats'][key] = self.frames['stats'][key][valid]
+                print("WARNING WARNING!: "+ str(len(valid) - np.sum(valid))+' soundings are filtered')
+
+        self.frames['stats']['records_all_stations_index'] = self.frames['stats']['records_all_stations_end_mod'].index
+
+
+        print("filtering stations from interface that have no records")
+        for STNID,station in self.frames['worldmap']['stations'].table.iterrows():
+            if ((self.frames['stats']['records_all_stations_index'].get_level_values('STNID')\
+                    == STNID).sum() == 0):
+                print("dropping", STNID)
+                self.frames['worldmap']['stations'].table = \
+                        self.frames['worldmap']['stations'].table.drop(STNID)
+                    
+        self.frames['worldmap']['stations_iterator'] = stations_iterator(self.frames['worldmap']['stations']) 
+        
+        # TO TEST: should be removed, since it's is also done just below
+        self.frames['stats']['stations_iterator'] = \
+            self.frames['worldmap']['stations_iterator'] 
+
+        self.frames['worldmap']['inputkey'] = self.frames['worldmap']['inputkeys'][0]
+        self.frames['worldmap']['inputkey'] = self.frames['worldmap']['inputkey']
+        self.next_station()
+
+        # self.goto_datetime_worldmap(
+        #     self.frames['profiles']['current_record_obs'].datetime.to_pydatetime(),
+        #     'after')
+    def sel_station(self,STNID=None,rownumber=None):
+
+        if (STNID is not None) and (rownumber is not None):
+            raise ValueError('Please provide either STNID or rownumber, not both.')
+
+        if (STNID is None) and (rownumber is None):
+            raise ValueError('Please provide either STNID or rownumber.')
+            
+        if STNID is not None:
+            self.frames['worldmap']['STNID'],\
+            self.frames['worldmap']['current_station'] \
+             = self.frames['worldmap']['stations_iterator'].set_STNID(STNID)
+            print(
+            self.frames['worldmap']['STNID'],\
+            self.frames['worldmap']['current_station'] \
+            )
+            self.update_station()
+        elif rownumber is not None:
+            self.frames['worldmap']['STNID'],\
+            self.frames['worldmap']['current_station'] \
+             = STNID,station = self.frames['worldmap']['stations_iterator'].set_row(rownumber)
+            self.update_station()
+
+
+    def next_station_event(self,event=None,**kwargs):
+        self.next_station(**kwargs)
+
+    def next_station(self,jump=1):
+        with suppress(StopIteration):
+            self.frames['worldmap']['STNID'],\
+            self.frames['worldmap']['current_station'] \
+                = self.frames['worldmap']['stations_iterator'].__next__(jump)
+            # self.frames['worldmap']['stations_iterator'].close()
+            # del(self.frames['worldmap']['stations_iterator'])
+            # self.frames['worldmap']['stations_iterator'] = \
+            #                 selfself.frames['worldmap']['stations'].iterrows()
+            # self.frames['worldmap']['STNID'],\
+            # self.frames['worldmap']['current_station'] \
+            #     = self.frames['worldmap']['stations_iterator'].__next__()
+
+        self.update_station()
+
+    def prev_station_event(self,event=None,**kwargs):
+        self.prev_station(**kwargs)
+
+    def prev_station(self):
+        self.next_station(jump = -1)
+    def update_station(self):
+        for key in ['STNID','current_station','stations_iterator']: 
+            self.frames['stats'][key] = self.frames['worldmap'][key] 
+
+
+
+        # generate index of the current station
+        self.frames['stats']['records_current_station_index'] = \
+            (self.frames['stats']['records_all_stations_index'].get_level_values('STNID')\
+             == \
+             self.frames['stats']['current_station'].name)
+
+        # create the value table of the records of the current station
+        tab_suffixes = \
+                ['_end_mod','_ini','_ini_pct']
+        if self.path_forcing is not None:
+            tab_suffixes=tab_suffixes+['_end_obs','_end_mod_stats','_end_obs_stats']
+
+        for tab_suffix in tab_suffixes:
+            self.frames['stats']['records_current_station'+tab_suffix] = \
+                self.frames['stats']['records_all_stations'+tab_suffix].iloc[self.frames['stats']['records_current_station_index']]
+
+        # go to first record of current station
+        self.frames['stats']['records_iterator'] = \
+                        records_iterator(self.frames['stats']['records_current_station_end_mod'])
+        (self.frames['stats']['STNID'] , \
+        self.frames['stats']['current_record_chunk'] , \
+        self.frames['stats']['current_record_index']) , \
+        self.frames['stats']['current_record_end_mod'] = \
+                        self.frames['stats']['records_iterator'].__next__()
+
+        for key in self.frames['stats'].keys():
+            self.frames['profiles'][key] = self.frames['stats'][key]
+
+        STNID = self.frames['profiles']['STNID']
+        chunk = self.frames['profiles']['current_record_chunk']
+        print(chunk)
+        if 'current_station_file_ini' in self.frames['profiles'].keys():
+            self.frames['profiles']['current_station_file_ini'].close()
+
+
+        fn_ini = format(STNID,"05d")+'_ini.yaml'
+        if not os.path.isfile(self.path_exp+'/'+fn_ini):
+            fn_ini = format(STNID,"05d")+'_'+str(chunk)+'_ini.yaml'
+
+        self.frames['profiles']['current_station_file_ini'] = \
+            open(self.path_exp+'/'+fn_ini,'r')
+
+        if 'current_station_file_end_mod' in self.frames['profiles'].keys():
+            self.frames['profiles']['current_station_file_end_mod'].close()
+
+
+        if  self.end_state:
+            self.frames['profiles']['current_station_file_end_mod'] = \
+                    open(self.path_exp+'/'+format(STNID,"05d")+'_'+str(chunk)+'_end.yaml','r')
+        else:
+            self.frames['profiles']['current_station_file_end_mod'] = \
+                    open(self.path_exp+'/'+fn_ini,'r')
+        if 'current_station_file_end_obs' in self.frames['profiles'].keys():
+            self.frames['profiles']['current_station_file_end_obs'].close()
+        if self.path_forcing is not None:
+            self.frames['profiles']['current_station_file_end_obs'] = \
+                open(self.path_forcing+'/'+format(STNID,"05d")+'_end.yaml','r')
+
+        # for the profiles we make a distinct record iterator, so that the
+        # stats iterator can move independently
+        self.frames['profiles']['records_iterator'] = \
+                        records_iterator(self.frames['profiles']['records_current_station_end_mod'])
+        (self.frames['profiles']['STNID'] , \
+        self.frames['profiles']['current_record_chunk'] , \
+        self.frames['profiles']['current_record_index']) , \
+        self.frames['profiles']['current_record_end_mod'] = \
+                        self.frames['profiles']['records_iterator'].__next__()
+
+
+        # for the profiles we make a distinct record iterator, so that the
+        # stats iterator can move independently
+
+        self.update_record()
+
+    def next_record_event(self,event=None,**kwargs):
+        self.next_record(**kwargs)
+
+    def next_record(self,jump=1):
+        
+        old_chunk =  self.frames['profiles']['current_record_chunk']
+
+        with suppress(StopIteration):
+            (self.frames['profiles']['STNID'] , \
+            self.frames['profiles']['current_record_chunk'] , \
+            self.frames['profiles']['current_record_index']) , \
+            self.frames['profiles']['current_record_end_mod'] = \
+                      self.frames['profiles']['records_iterator'].__next__(jump)
+        # except (StopIteration):
+        #     self.frames['profiles']['records_iterator'].close()
+        #     del( self.frames['profiles']['records_iterator'])
+        #     self.frames['profiles']['records_iterator'] = \
+        #                 self.frames['profiles']['records_current_station_end_mod'].iterrows()
+        #     (self.frames['profiles']['STNID'] , \
+        #     self.frames['profiles']['current_record_index']) , \
+        #     self.frames['profiles']['current_record_end_mod'] = \
+        #                     self.frames['profiles']['records_iterator'].__next__()
+
+        for key in self.frames['profiles'].keys():
+            self.frames['stats'][key] = self.frames['profiles'][key]
+
+        # chunk file has changed! So we need to open it!
+        if self.frames['profiles']['current_record_chunk'] != old_chunk:
+
+            STNID = self.frames['profiles']['STNID']
+            chunk = self.frames['profiles']['current_record_chunk']
+
+
+
+            if 'current_station_file_ini' in self.frames['profiles'].keys():
+                self.frames['profiles']['current_station_file_ini'].close()
+            self.frames['profiles']['current_station_file_ini'] = \
+                open(self.path_exp+'/'+format(STNID,"05d")+'_'+str(chunk)+'_ini.yaml','r')
+
+            if 'current_station_file_end_mod' in self.frames['profiles'].keys():
+                self.frames['profiles']['current_station_file_end_mod'].close()
+            self.frames['profiles']['current_station_file_end_mod'] = \
+                open(self.path_exp+'/'+format(STNID,"05d")+'_'+str(chunk)+'_end.yaml','r')
+
+            if self.path_forcing is not None:
+                if 'current_station_file_end_obs' in self.frames['profiles'].keys():
+                    self.frames['profiles']['current_station_file_end_obs'].close()
+                self.frames['profiles']['current_station_file_end_obs'] = \
+                    open(self.path_forcing+'/'+format(STNID,"05d")+'_end.yaml','r')
+
+        self.update_record()
+
+    def prev_record_event(self,event=None,**kwargs):
+        self.prev_record(**kwargs)
+
+    def prev_record(self,event=None):
+        self.next_record(jump=-1)
+
+    def update_record(self):
+        self.frames['profiles']['current_record_ini'] =  \
+            self.frames['profiles']['records_current_station_ini'].loc[\
+                  (self.frames['profiles']['STNID'] , \
+                  self.frames['profiles']['current_record_chunk'],\
+                  self.frames['profiles']['current_record_index'])]
+        if self.path_forcing is not None:
+            self.frames['profiles']['current_record_end_obs'] =  \
+                self.frames['profiles']['records_current_station_end_obs'].loc[\
+                      (self.frames['profiles']['STNID'] , \
+                      self.frames['profiles']['current_record_chunk'] , \
+                      self.frames['profiles']['current_record_index'])]
+
+            self.frames['profiles']['current_record_end_mod_stats'] = \
+                    self.frames['profiles']['records_all_stations_end_mod_stats'].loc[(\
+                        self.frames['profiles']['STNID'], \
+                        self.frames['profiles']['current_record_chunk'], \
+                        self.frames['profiles']['current_record_index'])]
+            self.frames['profiles']['current_record_end_obs_stats'] = \
+                    self.frames['profiles']['records_all_stations_end_obs_stats'].loc[(\
+                        self.frames['profiles']['STNID'],\
+                        self.frames['profiles']['current_record_chunk'],\
+                        self.frames['profiles']['current_record_index'])]
+        self.frames['profiles']['current_record_ini_pct'] = \
+                self.frames['profiles']['records_all_stations_ini_pct'].loc[(\
+                    self.frames['profiles']['STNID'],\
+                    self.frames['profiles']['current_record_chunk'],\
+                    self.frames['profiles']['current_record_index'])]
+
+        for key in self.frames['profiles'].keys():
+            self.frames['stats'][key] = self.frames['profiles'][key]
+        # frame
+        # note that the current station, record is the same as the stats frame for initialization
+
+        # select first 
+        #self.frames['profiles']['current_record_index'], \
+        #self.frames['profiles']['record_yaml_end_mod'] = \
+        #   get_record_yaml(self.frames['profiles']['current_station']['filename'],\
+        #                   self.frames['stats']['current_record_index'])
+        self.frames['profiles']['record_yaml_end_mod'] = \
+           get_record_yaml(
+               self.frames['profiles']['current_station_file_end_mod'], \
+               self.frames['profiles']['current_record_end_mod'].index_start,
+               self.frames['profiles']['current_record_end_mod'].index_end,
+               mode='model_output')
+                                
+        record_ini = self.frames['profiles']['records_all_stations_ini'].loc[
+                       (self.frames['stats']['STNID'] , \
+                        self.frames['stats']['current_record_chunk'] , \
+                        self.frames['stats']['current_record_index'])]
+
+        self.frames['profiles']['record_yaml_ini'] = \
+           get_record_yaml(
+               self.frames['profiles']['current_station_file_ini'], \
+               record_ini.index_start,
+               record_ini.index_end,
+                mode='model_input')
+
+        if self.path_forcing is not None:
+            record_end = self.frames['profiles']['records_all_stations_end_obs'].loc[
+                           (self.frames['stats']['STNID'] , \
+                            self.frames['stats']['current_record_chunk'] , \
+                            self.frames['stats']['current_record_index'])]
+
+            self.frames['profiles']['record_yaml_end_obs'] = \
+               get_record_yaml(
+                   self.frames['profiles']['current_station_file_end_obs'], \
+                   int(record_end.index_start),
+                   int(record_end.index_end),
+                    mode='model_input')
+
+
+        key = self.frames['worldmap']['inputkey']
+        # only redraw the map if the current world map has a time
+        # dimension
+        if (self.globaldata is not None) and ('time' in self.globaldata.datasets[key].page[key].dims):
+            self.goto_datetime_worldmap(
+                self.frames['profiles']['current_record_ini'].datetime.to_pydatetime(),
+                'after')
+            if "fig" in self.__dict__.keys():
+                self.refresh_plot_interface(only=['stats_lightupdate',
+                                                  'worldmap',
+                                                  'profiles'])
+        else:
+            if "fig" in self.__dict__.keys():
+                self.refresh_plot_interface(only=['stats_lightupdate',
+                                                  'worldmap',
+                                                  'profiles'])
+
+    def abline(self,slope, intercept,axis):
+        """Plot a line from slope and intercept"""
+        #axis = plt.gca()
+        x_vals = np.array(axis.get_xlim())
+        y_vals = intercept + slope * x_vals
+        axis.plot(x_vals, y_vals, 'k--')
+
+    def plot(self):
+        import pylab as pl
+        from matplotlib.widgets import Button
+        import matplotlib.pyplot as plt
+        import matplotlib as mpl
+        '''
+        Definition of the axes for the sounding table stats
+        '''
+        
+        fig = pl.figure(figsize=(14,9))
+        axes = {} #axes
+        btns = {} #buttons
+
+        # frames, which sets attributes for a group of axes, buttens, 
+        if self.path_forcing is not None:
+
+            for ikey,key in enumerate(list(self.frames['stats']['records_all_stations_end_mod_stats'].columns)):
+                label = 'stats_'+str(key)
+                axes[label] = fig.add_subplot(\
+                                len(self.frames['stats']['viewkeys']),\
+                                5,\
+                                5*ikey+1,label=label)
+                # Actually, the axes should be a part of the frame!
+                #self.frames['stats']['axes'] = axes[
+
+                # pointer to the axes' point data
+                axes[label].data = {}
+
+                # pointer to the axes' color fields
+                axes[label].fields = {}
+
+
+        fig.tight_layout()
+        fig.subplots_adjust(top=0.95,bottom=0.15,left=0.05,right=0.99,hspace=0.26,wspace=0.08)
+
+        label ='stats_colorbar'
+        axes[label] = fig.add_axes([0.025,0.06,0.18,0.025])
+        axes[label].fields = {}
+
+        from matplotlib.colors import LinearSegmentedColormap
+        cdictpres = {'blue': (\
+                           (0.,    0.,  0.),
+                           (0.25,  0.25, 0.25),
+                           (0.5,  .70, 0.70),
+                           (0.75, 1.0, 1.0),
+                           (1,     1.,  1.),
+                           ),
+               'green': (\
+                           (0. ,   0., 0.0),
+                           (0.25,  0.50, 0.50),
+                           (0.5,  .70, 0.70),
+                           (0.75,  0.50, 0.50),
+                           (1  ,    0,  0.),
+                           ),
+               'red':  (\
+                          (0 ,  1.0, 1.0),
+                          (0.25 ,  1.0, 1.0),
+                           (0.5,  .70, 0.70),
+                          (0.75 , 0.25, 0.25),
+                          (1,    0., 0.),
+                          )}
+        
+        self.statsviewcmap = LinearSegmentedColormap('statsviewcmap', cdictpres)
+
+
+        label = 'times'
+               
+        axes[label] = fig.add_axes([0.30,0.87,0.30,0.10]) #[*left*, *bottom*, *width*,    *height*]
+        # add pointers to the data of the axes
+        axes[label].data = {}
+        # add pointers to color fields (for maps and colorbars) in the axes
+        axes[label].fields = {}
+
+
+        label = 'worldmap'
+               
+        axes[label] = fig.add_axes([0.25,0.48,0.40,0.35]) #[*left*, *bottom*, *width*,    *height*]
+        # add pointers to the data of the axes
+        axes[label].data = {}
+        # add pointers to color fields (for maps and colorbars) in the axes
+        axes[label].fields = {}
+        axes[label].lat = None
+        axes[label].lon = None
+
+        
+        if self.globaldata is not None:
+            label = 'worldmap_colorbar'
+            axes[label] = fig.add_axes([0.25,0.44,0.40,0.05])
+            axes[label].fields = {}
+
+        # we make a overlying axes for the animations on the map, so that we don't need to redraw the whole map over and over again
+        label = 'worldmap_stations'
+        axes[label] = fig.add_axes([0.25,0.48,0.40001,0.350001]) #[*left*, *bottom*, *width*,    *height*]
+        axes[label].data = {}
+
+        fig.canvas.mpl_connect('pick_event', self.on_pick)
+        fig.canvas.callbacks.connect('motion_notify_event', self.on_plot_hover)
+
+
+        """ buttons definitions """
+        button_height = 0.055
+        button_hspace = 0.005
+        button_width  = 0.095
+        button_wspace = 0.005
+        buttons_upper = 0.28
+        buttons_left = 0.25
+
+        if self.globaldata is not None:
+            button_types = ['datetime','level','station','record']
+        else:
+            button_types = ['dataset','station','record']
+        
+        for ibutton_type,button_type in enumerate(button_types):
+            label='bprev'+button_type
+            axes[label] = fig.add_axes([
+                buttons_left,\
+                buttons_upper-ibutton_type*(button_height+button_hspace),\
+                button_width,\
+                button_height\
+                                                     ])
+            if button_type !='dataset':
+                btns[label] = Button(axes[label], 'Previous '+button_type)
+                btns[label].on_clicked(getattr(self, 'prev_'+button_type+'_event'))
+            else:
+                btns[label] = Button(axes[label], 'Previous input var')
+                btns[label].on_clicked(getattr(self, 'prev_'+button_type+'_event'))
+
+
+            label='bnext'+button_type
+            axes[label] = fig.add_axes([
+                buttons_left+button_width+button_wspace,\
+                buttons_upper-ibutton_type*(button_height+button_hspace),\
+                button_width,\
+                button_height\
+                                                     ])
+            if button_type !='dataset':
+                btns[label] = Button(axes[label], 'Next '+button_type)
+                btns[label].on_clicked(getattr(self, 'next_'+button_type+'_event'))
+            else:
+                btns[label] = Button(axes[label], 'Next input var')
+                btns[label].on_clicked(getattr(self, 'next_'+button_type+'_event'))
+
+        
+        # label = 'bprev_dataset'
+        # axes[label] = fig.add_axes([0.25,0.28,0.10,0.075])
+        # btns[label] = Button(axes[label], 'Previous dataset')
+        # btns[label].on_clicked(self.prev_dataset)
+
+        # label = 'bnext_dataset'
+        # axes[label] = fig.add_axes([0.35,0.28,0.10,0.075])
+        # btns[label] = Button(axes[label], 'Next dataset')
+        # btns[label].on_clicked(self.next_dataset)
+
+        # label = 'bprev_datetime'
+        # axes[label] = fig.add_axes([0.25,0.20,0.10,0.075])
+        # btns[label] = Button(axes[label], 'Previous datetime')
+        # btns[label].on_clicked(self.prev_datetime)
+
+        # label = 'bnext_datetime'
+        # axes[label] = fig.add_axes([0.35,0.20,0.10,0.075])
+        # btns[label] = Button(axes[label], 'Next datetime')
+        # btns[label].on_clicked(self.next_datetime)
+
+
+        # label = 'bprev_station'
+        # axes[label] = fig.add_axes([0.25,0.12,0.10,0.075])
+        # btns[label] = Button(axes[label], 'Previous station')
+        # btns[label].on_clicked(self.prev_station)
+
+        # label = 'bnext_station'
+        # axes[label] = fig.add_axes([0.35,0.12,0.10,0.075])
+        # btns[label] = Button(axes[label], 'Next station')
+        # btns[label].on_clicked(self.next_station)
+
+        # label = 'bprev_record'
+        # axes[label] = fig.add_axes([0.25,0.04,0.10,0.075])
+        # btns[label] = Button(axes[label], 'Previous record')
+        # btns[label].on_clicked(self.prev_record)
+
+        # label = 'bnext_record'
+        # axes[label] = fig.add_axes([0.35,0.04,0.10,0.075])
+        # btns[label] = Button(axes[label], 'Next record')
+        # btns[label].on_clicked(self.next_record)
+
+
+        # self.nstatsview = nstatsview
+        # self.statsviewcmap = statsviewcmap
+        self.fig = fig
+        self.axes = axes
+        self.btns = btns
+        self.tbox = {}
+        # self.hover_active = False
+
+        #self.tbox['loading'] = fig.text(0.30,0.01, " ",fontsize=10, 
+        #                                transform=plt.gcf().transFigure)
+
+        self.tbox['datetime'] =  fig.text(0.70, 0.96, " ", fontsize=10,
+                                          transform=plt.gcf().transFigure)
+
+        label = 'air_ap:theta'
+        self.axes[label] = fig.add_axes([0.70,0.44,0.12,0.50], label=label)
+
+        label = 'air_ap:q'
+        self.axes[label] = fig.add_axes([0.86,0.44,0.12,0.50], label=label)
+
+        label = 'out:h'
+        self.axes[label] = fig.add_axes([0.50,0.27,0.22,0.09], label=label)
+
+        label = 'out:theta'
+        self.axes[label] = fig.add_axes([0.50,0.17,0.22,0.09], label=label)
+
+        label = 'out:q'
+        self.axes[label] = fig.add_axes([0.50,0.07,0.22,0.09], label=label)
+
+        label = 'SEB'
+        self.axes[label] = fig.add_axes([0.77,0.07,0.22,0.30], label=label)
+
+
+        self.hover_active = False
+        self.fig = fig
+        self.fig.show()
+        self.fig.canvas.draw()
+        self.refresh_plot_interface()
+
+
+    # def scan_stations(self):
+    #     blabla
+        
+
+
+    # def get_records(current_file):
+    #     records = pd.DataFrame()
+
+    #     # initial position
+    #     next_record_found = False
+    #     while(not next_record_found):
+    #         next_record_found = (current_file.readline() == '---\n')
+    #     next_tell = current_file.tell() 
+    #     end_of_file = (currentline == '') # an empty line means we are at the end
+
+    #     while not end_of_file:
+    #         current_tell = next_tell
+    #         next_record_found = False
+    #         current_file.seek(current_tell)
+    #         while ( (not next_record_found) and (not end_of_file)):
+    #             current_line = current_file.readline()
+    #             next_record_found = (currentline == '---\n')
+    #             end_of_file = (currentline == '') # an empty line means we are at the end
+
+    #         # we store the position of the next record
+    #         next_tell = current_file.tell() 
+    #         
+    #         # we get the current record. Unfortunately we need to reset the
+    #         # yaml record generator first.
+    #         current_yamlgen.close()
+    #         current_yamlgen = yaml.load_all(current_file)
+    #         current_file.seek(current_tell)
+    #         current_record_end_mod = current_yamlgen.__next__()
+    #     current_yamlgen.close()
+
+    #     return records
+
+       #      next_record_found = False
+       #      while(not record):
+       #          next_record_found = (self.current_file.readline() == '---\n')
+       #      self.current_tell0 = self.current_file.tell() 
+
+       #  
+
+       #  next_record_found = False
+       #  while(not next_record_found):
+       #      next_record_found = (self.current_file.readline() == '---\n')
+       #  self.current_tell0 = self.current_file.tell() 
+
+       #  next_record_found = False
+       #  while(not next_record_found):
+       #      next_record_found = (self.current_file.readline() == '---\n')
+       #  self.current_tell1 = self.current_file.tell() 
+
+
+       #  self.current_yamlgen.close()
+       #  self.current_yamlgen = yaml.load_all(self.current_file)
+       #  self.current_file.seek(self.current_tell0)
+       #  self.r0 = self.current_yamlgen.__next__()
+
+       #  self.current_file.seek(self.current_tell1)
+       #  next_record_found = False
+       #  while ( (not next_record_found) and (not end_of_file):
+       #      current_line = self.current_file.readline()
+       #      next_record_found = (currentline == '---\n')
+       #      end_of_file = (currentline == '') # an empty line means we are at the end
+
+       #  self.current_tell2 = self.current_file.tell() 
+
+
+       #  self.current_yamlgen.close()
+       #  self.current_yamlgen = yaml.load_all(self.current_file)
+       #  self.current_file.seek(self.current_tell1)
+       #  self.r1 = self.current_yamlgen.__next__()
+
+       #  self.current_file.seek(self.current_tell2)
+       #  next_record_found = False
+       #  while(not next_record_found):
+       #      next_record_found = (self.current_file.readline() == '---\n')
+       #  self.current_tell3 = self.current_file.tell() 
+
+       #  self.current_yamlgen.close()
+       #  self.current_yamlgen = yaml.load_all(self.current_file)
+       #  self.current_file.seek(self.current_tell2)
+       #  self.r2 = self.current_yamlgen.__next__()
+
+       #  # go to position of next record in file
+       #  self.current_file.seek(self.current_tell3)
+       #  next_record_found = False
+       #  while(not next_record_found):
+       #      next_record_found = (self.current_file.readline() == '---\n')
+       #  self.current_tell4 = self.current_file.tell() 
+
+       #  self.current_yamlgen.close()
+       #  self.current_yamlgen = yaml.load_all(self.current_file)
+       #  self.current_file.seek(self.current_tell3)
+       #  self.r3 = self.current_yamlgen.__next__()
+ 
+       #  #self.update_tablestats(SOUNDINGS_TABLESTATS)
+
+    def goto_datetime_worldmap(self,DT,shift=None):
+        DT = np.datetime64(DT) #self.globaldata.datasets[self.axes['worldmap'].focus['key']].variables['time'].values[self.axes['worldmap'].focus['iDT']]
+        if self.globaldata is not None:
+            if 'time' in self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables[self.frames['worldmap']['inputkey']].dims:
+                self.globaldata.datasets[self.frames['worldmap']['inputkey']].browse_page(time=DT)
+                DIST = np.abs((self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables['time'].values - DT))
+                self.frames['worldmap']['iDT'] = np.where((DIST) == np.min(DIST))[0][0]
+                if ((shift == 'after') and (self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables['time'].values[self.frames['worldmap']['iDT']] < DT)):
+                    self.frames['worldmap']['iDT'] += 1
+                elif ((shift == 'before') and (self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables['time'].values[self.frames['worldmap']['iDT']] > DT)):
+                    self.frames['worldmap']['iDT'] -= 1 
+                # for gleam, we take the values of the previous day
+                if self.frames['worldmap']['inputkey'] in ['wg','w2']:
+                    self.frames['worldmap']['iDT'] -= 2 
+                self.frames['worldmap']['DT'] = self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables['time'].values[self.frames['worldmap']['iDT']]
+            #else:
+            #    self.frames['worldmap'].pop('DT')
+
+    def next_datetime_event(self,event=None,**kwargs):
+        self.next_datetime(**kwargs)
+    def next_datetime(self,event=None):
+        if 'time' in self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables[self.frames['worldmap']['inputkey']].dims:
+            # for now we don't go to different files, so we cannot go to
+            # another file 
+            self.frames['worldmap']['iDT'] = (self.frames['worldmap']['iDT'] + 1) % len(self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables['time'].values)
+            self.frames['worldmap']['DT'] = self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables['time'].values[self.frames['worldmap']['iDT']]
+            if "fig" in self.__dict__.keys():
+                self.refresh_plot_interface(only='worldmap') 
+
+    def prev_datetime_event(self,event=None,**kwargs):
+        self.prev_datetime(**kwargs)
+    def prev_datetime(self,event=None):
+        if 'time' in self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables[self.frames['worldmap']['inputkey']].dims:
+            # for now we don't go to different files, so we cannot go to
+            # another file 
+            self.frames['worldmap']['iDT'] = (self.frames['worldmap']['iDT'] - 1) % len(self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables['time'].values)
+            self.frames['worldmap']['DT'] = self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.variables['time'].values[self.frames['worldmap']['iDT']]
+            if "fig" in self.__dict__.keys():
+                self.refresh_plot_interface(only='worldmap') 
+
+    def next_dataset_event(self,event=None,**kwargs):
+        self.next_dataset(**kwargs)
+    def next_dataset(self,event=None):
+        ikey = self.frames['worldmap']['inputkeys'].index(self.frames['worldmap']['inputkey'])
+        ikey = (ikey + 1) % len(self.frames['worldmap']['inputkeys'])
+        self.sel_dataset(self.frames['worldmap']['inputkeys'][ikey])
+    def prev_dataset_event(self,event=None,**kwargs):
+        self.prev_dataset(**kwargs)
+    def prev_dataset(self,event=None):
+        ikey = self.frames['worldmap']['inputkeys'].index(self.frames['worldmap']['inputkey'])
+        ikey = (ikey - 1) % len(self.frames['worldmap']['inputkeys'])
+        self.sel_dataset(self.frames['worldmap']['inputkeys'][ikey])
+
+    def sel_dataset(self,inputkey):
+        self.frames['worldmap']['inputkey'] = inputkey
+        self.frames['stats']['inputkey'] = self.frames['worldmap']['inputkey'] # this is used for showing the percentiles per station in color.
+        self.goto_datetime_worldmap(
+            self.frames['profiles']['current_record_ini'].datetime.to_pydatetime(),
+            'after')# get nearest datetime of the current dataset to the profile
+
+        print('seldata0')
+
+        if 'level' not in self.frames['worldmap'].keys():
+
+            if self.globaldata is not None:
+                if 'lev' in list(self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.dims):
+                    levels = self.globaldata.datasets[self.frames['worldmap']['inputkey']].page['lev']
+                else:
+                    levels = np.array([0])
+            else:
+                levels = np.array([0])
+
+            self.frames['worldmap']['level'] = np.max(levels)
+            print('seldata1')
+
+            minlev = np.min(levels)
+            maxlev = np.max(levels)
+            curlev = self.frames['worldmap']['level']
+            curlev = np.max([curlev,np.min(levels)])
+            curlev = np.min([curlev,np.max(levels)])
+            print('seldata2')
+
+            self.frames['worldmap']['level'] = curlev
+            print('seldata3')
+
+
+            print('seldata4')
+        self.sel_level(self.frames['worldmap']['level'])
+
+
+
+    def sel_level(self,level):
+
+        # if 'lev' not in list(self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.dims):
+        #     raise ValueError('lev dimension not in dataset '+self.frames['worldmap']['inputkey'])
+
+        print('seldata5')
+
+        if self.globaldata is not None:
+            if 'lev' in list(self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.dims):
+                if level > (np.max(self.globaldata.datasets[self.frames['worldmap']['inputkey']].page['lev'])):
+                    raise ValueError('Level '+str(level)+' exceed those of the current dataset: '+str(self.globaldata.datasets[frames['worldmap']['inputkey']].page['lev']))
+                if level < (np.min(self.globaldata.datasets[self.frames['worldmap']['inputkey']].page['lev'])):
+                    raise ValueError('Level '+str(level)+' is lower than those of the current dataset: '+str(self.globaldata.datasets[frames['worldmap']['inputkey']].page['lev']))
+                print('seldata6')
+                self.frames['worldmap']['level'] = level
+
+            print(level)
+        if "fig" in self.__dict__.keys():
+            self.refresh_plot_interface(only=['worldmap','stats_lightupdate','stats_colorbar']) 
+
+        print('seldata7')
+
+    def next_level_event(self,event=None,**kwargs):
+        self.next_level(self,**kwargs)
+    def next_level(self,event=None,jump=1):
+        if 'lev' not in list(self.globaldata.datasets[self.frames['worldmap']['inputkey']].page.dims.keys()):
+            raise ValueError('lev dimension not in dataset'+self.frames['worldmap']['inputkey'])
+        levels =  self.globaldata.datasets[self.frames['worldmap']['inputkey']].page['lev']
+        level = self.frames['worldmap']['level']
+        level =  ((level + jump - min(levels)) % (max(levels)-min(levels))) + min(levels)
+        self.sel_level(level)
+
+    def prev_level_event(self,event=None,**kwargs):
+        self.prev_level(self,**kwargs)
+    def prev_level(self):
+        self.next_level(jump=-1)
+
+        #self.frames['worldmap']['level'] = level: 
+       
+    # def prev_station(self,event=None):
+    #     self.istation = (self.istation - 1) % self.stations.shape[0]
+    #     self.update_station()
+
+
+
+
+    #def update_datetime(self):
+    #    if 'time' in self.globaldata.datasets[self.worldmapfocus['key']].variables[self.worldmapfocus['key']].dims:
+    #    #if 'time' in list(dict(self.globaldata.datasets[self.worldmapfocus['key']].variables[self.worldmapfocus['key']].dims).keys()):
+    #        #self.worldmapfocus['DT'] = self.globaldata.datasets[self.worldmapfocus['key']].variables['time'].values[self.worldmapfocus['iDT']]
+    #        print(self.worldmapfocus['DT'])
+    #        self.refresh_plot_interface(only='worldmap')
+
+    def refresh_plot_interface(self,only=None,statsnewdata=True,**args):
+
+        #print('r1')
+        for argkey in args.keys():
+            self.__dict__[arg] = args[argkey]
+
+        axes = self.axes
+        tbox = self.tbox
+        frames = self.frames
+        fig = self.fig
+ 
+        if self.globaldata is not None:
+            if (only is None) or ('worldmap' in only):
+                globaldata = self.globaldata
+                print('hello0')
+                if 'time' in globaldata.datasets[frames['worldmap']['inputkey']].page.variables[frames['worldmap']['inputkey']].dims:
+                    globaldata.datasets[frames['worldmap']['inputkey']].browse_page(time=frames['worldmap']['DT'])
+                    datasetxr = globaldata.datasets[frames['worldmap']['inputkey']].page.isel(time = frames['worldmap']['iDT'])
+                else:
+                    datasetxr = globaldata.datasets[frames['worldmap']['inputkey']].page
+                if 'lev' in datasetxr.dims:
+                    datasetxr = datasetxr.isel(lev=self.frames['worldmap']['level'])
+                keystotranspose = ['lat','lon']
+                for key in dict(datasetxr.dims).keys():
+                    if key not in keystotranspose:
+                        keystotranspose.append(key)
+
+                datasetxr = datasetxr.transpose(*keystotranspose)
+                datasetxr = datasetxr.sortby('lat',ascending=False)
+                print('hello1')
+
+                lonleft = datasetxr['lon'].where(datasetxr.lon > 180.,drop=True) 
+                lonleft = lonleft - 360.
+                print('hello2')
+                lonright = datasetxr['lon'].where(datasetxr.lon <= 180.,drop=True) 
+                label = 'worldmap'
+                axes[label].clear()
+                axes[label].lon = xr.concat([lonleft,lonright],'lon').values
+                axes[label].lat = np.sort(globaldata.datasets[frames['worldmap']['inputkey']].page.variables['lat'].values)[::-1] #sortby('lat',ascending=False).values
+                print('hello3')
+
+            #if 'axmap' not in self.__dict__ :
+            #    self.axmap = self.fig.add_axes([0.39,0.5,0.34,0.5])
+            #else:
+
+            #stations = self.stations
+
+
+            # self.gmap = Basemap(projection='kav7', lat_0 = 0, lon_0 =0,
+            #     resolution = 'l', 
+            # area_thresh = 0.1,
+            #     llcrnrlon=-180., llcrnrlat=-90.0,
+            #     urcrnrlon=180., urcrnrlat=90.0,ax=self.axmap)
+            # 
+            # self.gmap.drawcoastlines(color='white',linewidth=0.3)
+            # self.gmap.drawcountries(color='white',linewidth=0.3)
+            # #self.gmap.fillcontinents(color = 'gray')
+            # self.gmap.drawmapboundary(color='white',linewidth=0.3)
+            # # self.gmap.drawmeridians(np.arange(-180, 180+45, 60.),labels=[1,1,0,1])
+            # # self.gmap.drawparallels(np.arange(-90, 90, 30.),labels=[1,0,0,0])
+            # self.gmap.drawmeridians(np.arange(-180, 180+45, 60.),color='white',linewidth=0.3,labels=[0,0,0,0])
+            # self.gmap.drawparallels(np.arange(-90, 90, 30.),color='white',linewidth=0.3,labels=[0,0,0,0])
+            # #self.ax5.shadedrelief()
+
+           #if 'time' in list(dict(self.datasets[self.axes['worldmap'].focus['key']].variables[self.axes['worldmap'].focus['key']].dims).keys()):
+
+
+                fieldleft =  datasetxr[frames['worldmap']['inputkey']].where(datasetxr.lon > 180.,drop=True) 
+                fieldright = datasetxr[frames['worldmap']['inputkey']].where(datasetxr.lon <= 180.,drop=True) 
+                print('hello4')
+
+                field =xr.concat([fieldleft,fieldright],'lon') #.sortby('lat',ascending=False).values
+
+                #np.concatenate([viewframe.datasets['cc']['cc'].page.isel(time=0).where(viewframe.datasets['cc'].lon > 180).values,viewframe.datasets['cc']['cc'].isel(time=0).where(viewframe.datasets['cc'].lon <= 180).values],axis=1)
+                axes[label].axis('off')
+                print('hello5')
+
+                from matplotlib import cm
+                axes[label].fields[label] = axes[label].imshow(field[:,:],interpolation='none',cmap = cm.viridis )
+                
+                print('hello6')
+                title=frames['worldmap']['inputkey']
+                if globaldata is not None: 
+                    if 'time' in globaldata.datasets[frames['worldmap']['inputkey']].page.variables[frames['worldmap']['inputkey']].dims:
+                        title = title+' ['+pd.to_datetime(frames['worldmap']['DT']).strftime("%Y/%m/%d %H:%M") +'UTC]'
+                axes[label].set_title(title)
+                print('hello7')
+
+                label ='worldmap_colorbar'
+                axes[label].clear()
+                axes[label].fields[label] = fig.colorbar(axes['worldmap'].fields['worldmap'],cax=axes[label],orientation='horizontal',label=frames['worldmap']['inputkey']+' ['+self.units[frames['worldmap']['inputkey']]+']')
+
+
+                # lons, lats = np.meshgrid(axes[label].lon,axes[label].lat)
+                # x,y = self.gmap(lons,lats)
+                # #self.cont_map = self.axmap.contourf(x,y,field.T,cmap=gmapcm)
+                # self.cont_map = self.axmap.pcolormesh(x,y,field.T,cmap=gmapcm)
+        else:
+            # simplified version for only showing a simple image
+            label = 'worldmap'
+            axes[label].lat = np.arange(-90.,91.,1.)[::-1]
+            axes[label].lon = np.arange(-180.,181.,1.)[:]
+            axes[label].fields[label] = axes[label].imshow(img_worldmap)
+            axes[label].axis('off')
+
+
+
+        if (self.path_forcing is not None) and \
+           (self.frames['worldmap']['inputkey'] in self.frames['stats']['records_all_stations_ini_pct'].keys()) and \
+           (self.path_forcing is not None) and \
+           ((only is None) or ('stats' in only) or ('stats_lightupdate' in only)):
+
+            statskeys_out = list(self.frames['stats']['records_all_stations_end_mod_stats'].columns)
+            store_xlim = {}
+            store_ylim = {}
+            for ikey, key in enumerate(statskeys_out):
+                if (only is not None) and ('stats_lightupdate' in only):
+                    store_xlim[key] = axes['stats_'+key].get_xlim()
+                    store_ylim[key] = axes['stats_'+key].get_ylim()
+                self.axes['stats_'+key].clear()    
+
+            label = 'times'
+            self.axes[label].clear()
+
+            key = 'dthetadt'
+            x = self.frames['stats']['records_all_stations_ini']['datetime']
+            #print(x)
+            y = self.frames['stats']['records_all_stations_end_obs_stats'][key]
+            #print(y)
+            z = self.frames['stats']['records_all_stations_ini_pct'][self.frames['worldmap']['inputkey'] ]
+            #print(z)
+
+            alpha_cloud_pixels = 1./(1.+1./(0.15 * 10000. / len(self.frames['stats']['records_all_stations_end_mod'])))
+            self.axes[label].data[label] = self.axes[label].scatter(x.values,
+                                                                    y.values,
+                                                                    c=z.values,
+                                                                    cmap=self.statsviewcmap,
+                                                                    s=2,
+                                                                    vmin=0.,
+                                                                    vmax=1.,
+                                                                    alpha=alpha_cloud_pixels)
+
+            
+            x = self.frames['stats']['records_current_station_ini']['datetime']
+            y = self.frames['stats']['records_current_station_end_obs_stats'][key]
+            z = self.frames['stats']['records_current_station_ini_pct'][self.frames['worldmap']['inputkey'] ]
+            self.axes[label].data[label+'_current_station_hover'] = self.axes[label].scatter(x.values,y.values,c=z.values,cmap=self.statsviewcmap,s=5,picker=5,vmin=0.,vmax=1.,edgecolor='k',linewidth=0.3)
+
+
+            x = self.frames['profiles']['records_current_station_ini']['datetime']
+            y = self.frames['profiles']['records_current_station_end_obs_stats'][key]
+            z = self.frames['profiles']['records_current_station_ini_pct'][self.frames['worldmap']['inputkey'] ]
+
+            self.axes[label].data[label+'_current_station'] = self.axes[label].scatter(x.values,y.values,c=z.values,cmap=self.statsviewcmap,s=20,picker=20,vmin=0.,vmax=1.,edgecolor='k',linewidth=0.8)
+
+            self.axes[label].set_xlim((dt.datetime(1981,1,1),dt.datetime(2018,1,1)))
+            self.axes[label].set_ylabel(key+ ' ['+self.units[key]+']')
+
+            for ikey, key in enumerate(statskeys_out):
+
+                # show data of all stations
+                x = self.frames['stats']['records_all_stations_end_obs_stats'][key]
+                y = self.frames['stats']['records_all_stations_end_mod_stats'][key]
+                z = self.frames['stats']['records_all_stations_ini_pct'][self.frames['worldmap']['inputkey'] ]
+                qvalmax = x.quantile(0.999)
+                qvalmin = x.quantile(0.001)
+                print('applying extra filter over extreme values for plotting stats')
+                selx = (x >= qvalmin) & (x < qvalmax)
+                sely = (x >= qvalmin) & (x < qvalmax)
+                x = x[selx & sely]
+                y = y[selx & sely]
+                z = z[selx & sely]
+                self.axes['stats_'+key].data['stats_'+key] = \
+                       self.axes['stats_'+key].scatter(x,y, c=z,\
+                                cmap=self.statsviewcmap,\
+                                s=3,picker=3,label=key,vmin=0.,vmax=1.,alpha=alpha_cloud_pixels)
+
+                if len(x) > 1:
+                    fit = np.polyfit(x, y, deg=1)
+                    self.axes['stats_'+key].data['stats_'+key+'_fit'] = \
+                         self.axes['stats_'+key].plot(x, fit[0] * x + fit[1], color='k',alpha=0.4,lw=4)
+
+                x = self.frames['stats']['records_current_station_end_obs_stats'][key]
+                y = self.frames['stats']['records_current_station_end_mod_stats'][key]
+                z = self.frames['stats']['records_current_station_ini_pct'][self.frames['worldmap']['inputkey'] ]
+                self.axes['stats_'+key].data['stats_'+key+'_current_station_hover'] = \
+                       self.axes['stats_'+key].scatter(x.values,y.values, c=z.values,\
+                                cmap=self.statsviewcmap,\
+                                s=10,picker=10,label=key,vmin=0.,vmax=1.,edgecolor='k',linewidth=0.3)
+
+                x = self.frames['profiles']['records_current_station_end_obs_stats'][key]
+                y = self.frames['profiles']['records_current_station_end_mod_stats'][key]
+                z = self.frames['profiles']['records_current_station_ini_pct'][self.frames['worldmap']['inputkey'] ]
+                self.axes['stats_'+key].data['stats_'+key+'_current_station'] = \
+                       self.axes['stats_'+key].scatter(x.values,y.values, c=z.values,\
+                                cmap=self.statsviewcmap,\
+                                s=20,picker=20,label=key,vmin=0.,vmax=1.,edgecolor='k',linewidth=0.8)
+
+                if len(x) > 1:
+                    fit = np.polyfit(x, y, deg=1)
+                    self.axes['stats_'+key].data['stats_'+key+'_fit'] = \
+                         self.axes['stats_'+key].plot(x, fit[0] * x + fit[1], color='k',alpha=0.8,lw=3)
+
+                x = self.frames['stats']['current_record_end_obs_stats'][key]
+                y = self.frames['stats']['current_record_end_mod_stats'][key]
+                z = self.frames['stats']['current_record_ini_pct'][self.frames['worldmap']['inputkey'] ]
+
+                text = 'EXT: '+ format(x,'2.4f')+ ', MOD: ' + format(y,'2.4f')
+                self.axes['stats_'+key].data['stats_'+key+'_current_record'] = \
+                    axes['stats_'+key].annotate(text, \
+                                               xy=(x,y),\
+                                               xytext=(0.05,0.05),\
+                                               textcoords='axes fraction',\
+                                               bbox=dict(boxstyle="round",fc=self.statsviewcmap(z),edgecolor='black'),\
+                                               color='white',\
+                                               arrowprops=dict(arrowstyle="->",linewidth=1.1,color='black'))
+                # self.axes['stats_'+key].data[key+'_current_record'] = \
+                #        self.axes['stats_'+key].scatter(x,y, c=z,\
+                #                 cmap=self.statsviewcmap,\
+                #                 s=30,picker=15,label=key,vmin=0.,vmax=1.,edgecolor='k',linewidth=1.1)
+
+                # axes['stats_'+key].set_title('relative deviation per station of '+ key)
+                self.axes['stats_'+key].set_title(key+ ' ['+self.units[key]+']')
+                # # highlight data for curent station
+                # self.frames['stats']['records_all_stations_end_mod_stats'].iloc[self.frames['stats']['records_all_stations_index'].get_level_values('STNID') == self.frames['stats']['current_station'].name]
+
+                #text = 'EXT: '+format(seltablestatsstdrel_statannotate[key+'_ext'],'2.4f')+ ', MOD: '+format(seltablestatsstdrel_statannotate[key+'_end_mod'],'2.4f')
+
+                if ikey == len(statskeys_out)-1:
+                    self.axes['stats_'+key].set_xlabel('external')
+                    #axes[label].set_xlabel('ext: '+ key+' ['+statsunits[ikey]+']')
+                axes['stats_'+key].set_ylabel('model')
+
+
+                if (only is not None) and ('stats_lightupdate' in only):
+                    self.axes['stats_'+key].set_xlim(*store_xlim[key])
+                    self.axes['stats_'+key].set_ylim(*store_ylim[key])
+                else:
+                    limlow = np.min((axes['stats_'+key].get_xlim()[0],axes['stats_'+key].get_ylim()[0]))
+                    limhigh = np.max((axes['stats_'+key].get_xlim()[1],axes['stats_'+key].get_ylim()[1]))
+                    self.axes['stats_'+key].set_xlim(limlow,limhigh)
+                    self.axes['stats_'+key].set_ylim(limlow,limhigh)
+                self.abline(1,0,axis=self.axes['stats_'+key])
+
+        if (only is None) or ('stats_colorbar' in only):
+            label ='stats_colorbar'
+            axes[label].clear()
+            import matplotlib as mpl
+            norm = mpl.colors.Normalize(vmin=0.,vmax=1.)
+            self.axes[label].fields[label] = \
+             mpl.colorbar.ColorbarBase(self.axes[label],\
+                        orientation='horizontal',\
+                        label="percentile of "+self.frames['worldmap']['inputkey'],
+                        alpha=1.,
+                                cmap=self.statsviewcmap,\
+                                       norm=norm
+                         )
+
+        #print('r1')
+        if (only is None) or ('worldmap' in only) or ('worldmap_stations' in only):
+            label = 'worldmap_stations'
+            axes[label].clear()
+            #print('r2')
+            stations = self.frames['worldmap']['stations'].table
+            globaldata = self.globaldata
+            
+            key = label
+
+            #print('r3')
+            if (stations is not None):
+                xlist = []
+                ylist = []
+                #print('r4')
+                for iSTN,STN in frames['worldmap']['stations'].table.iterrows():
+            #        x,y =self.gmap(STN['longitude'],STN['latitude'])
+            #        self.gmap.plot(x,y, 'mo' if (self.STNID == STN['ID']) else 'ro' , markersize=1)
+                    x,y = len(axes['worldmap'].lon)*(STN['longitude']- axes['worldmap'].lon[0])/(axes['worldmap'].lon[-1] - axes['worldmap'].lon[0]) ,len(axes['worldmap'].lat)*(STN['latitude']- axes['worldmap'].lat[0])/(axes['worldmap'].lat[-1] - axes['worldmap'].lat[0])
+                    xlist.append(x)
+                    ylist.append(y)
+                #picker is needed to make it clickable (pick_event)
+                print(label)
+                axes[label].data[label] = axes[label].scatter(xlist,ylist,
+                                                              c='r', s=15,
+                                                              picker = 15,
+                                                              label=key,
+                                                              edgecolor='k',
+                                                              linewidth=0.8)
+
+            # cb.set_label('Wilting point [kg kg-3]')
+                #print('r5')
+
+                
+            #     xseries = []
+            #     yseries = []
+            #     for iSTN,STN in stations.iterrows():
+            # #        x,y =self.gmap(STN['longitude'],STN['latitude'])
+            # #        self.gmap.plot(x,y, 'mo' if (self.STNID == STN['ID']) else 'ro' , markersize=1)
+            #         x,y = len(axes[label].lon)*(STN['longitude_ext']- axes[label].lon[0])/(axes[label].lon[-1] - axes[label].lon[0])  ,len(axes[label].lat)*(STN['latitude_ext']- axes[label].axes[label].lat[0])/(axes[label].lat[-1] - axes[label].axes[label].lat[0])
+            #         xseries.append(x)                    
+            #         yseries.append(y)
+            #         
+            #         
+            #     axes[label].data[label] = axes[label].scatter(xseries,yseries, c='r' , s=15, edgecolor='none',label=key)
+                    
+                if ('current_station' in frames['worldmap']):
+                    #print('r5')
+                    STN = frames['stats']['current_station']
+                    STNID = frames['stats']['STNID']
+                    #print('r5')
+
+                    x,y = len(axes['worldmap'].lon)* \
+                            (STN['longitude']- axes['worldmap'].lon[0])/(axes['worldmap'].lon[-1] - axes['worldmap'].lon[0]),\
+                          len(axes['worldmap'].lat)* \
+                            (STN['latitude']- axes['worldmap'].lat[0])/(axes['worldmap'].lat[-1] - axes['worldmap'].lat[0])
+                    #print('r6')
+                    #VAL = self.seltablestats[(self.seltablestats['STNID'] \
+                    #                          == \
+                    #                          self.frames['worldmap']['STNID'])\
+                    #                         & \
+                    #                         (self.seltablestats['DT'] \
+                    #                          == self.axes['statsview0].focus['DT']) \
+                    #                        ][self.axes['worldmap'].focus['key']+'_ext'].iloc[0]
+                    #print('r7')
+                    text = 'STNID: '+ format(STNID,'10.0f') + \
+                            ', LAT: '+format(STN['latitude'],'3.3f')+ \
+                            ', LON: '+format(STN['longitude'],'3.3f')+ \
+                            ', #SOUNDINGS: '+str(self.frames['stats']['records_current_station_end_mod'].shape[0]) \
+
+                            #+', VAL: '+format(VAL,'.3e')
+
+                    axes[label].scatter(x,y, c='r', s=30,\
+                                        edgecolor='k',picker=30,label=key,linewidth=1.1)
+                    #print('r8')
+            
+                    #colorrange = list(axes[label].fields['worldmap'].get_clim())
+                    #colorstation = (VAL-colorrange[0])/(colorrange[1]-colorrange[0])
+                    #colorstation = max((min((1.,colorstation)),0.))
+                    colorstation =0.2
+                    from matplotlib import cm
+                    axes[label].annotate(text,
+                                         xy=(x,y),
+                                         xytext=(0.05,0.05),
+                                         textcoords='axes fraction', 
+                                         bbox=dict(boxstyle="round",
+                                         fc = cm.viridis(colorstation),edgecolor='black'),
+                                         arrowprops=dict(arrowstyle="->",
+                                                         linewidth=1.1,color='black'),
+                                         color='white' if colorstation < 0.5 else 'black')
+                    #print('r9')
+
+                    # #pos = sc.get_offsets()[ind["ind"][0]]
+                    # 
+                    # axes[label.data[label+'statannotate'].xy = (seltablestatsstdrel_statannotate[key+'_ext'],seltablestatsstdrel_statannotate[key+'_end_mod'])
+                    # text = 'STN: '+str(int(axes['statsview0'].focus['STNID']))+', DT: '+str(axes['statsview0'].focus['DT'])+', EXT: '+str(seltablestatsstdrel_statannotate[key+'_ext'])+', MOD: '+str(seltablestatsstdrel_statannotate[key+'_end_mod'])
+                    # axes[label].data[label+'statannotate'].set_text(text)
+                    #axes[label].data[label+'statannotate'].get_bbox_patch().set_facecolor(statsviewcmap(seltablestatspct_statannotate[cmapkey]))
+                    # axes[label].data[label+'statannotate'].get_bbox_patch().set_alpha(0.4)
+            #print('r9')
+            axes[label].axis('off')
+            axes[label].set_xlim(0,(len(axes['worldmap'].lon)))
+            axes[label].set_ylim((len(axes['worldmap'].lat),0))
+            #print('r10')
+
+        if (only is None) or ('profiles' in only): 
+            #print('r11')
+
+            # # self.istation = np.where(self.stations['ID'] == STNID)[0][0]
+            # # self.update_station(goto_first_sounding=False)
+            # isounding = np.where(pd.DatetimeIndex(self.df_soundings_eval_pairs.datetime) == self.profilefocus['DT'])[0][0]
+            # #self.isounding = (self.isounding - 1) % self.df_soundings_eval_pairs.shape[0]
+            # self.morning_sounding = self.df_soundings_eval_pairs.loc[isounding]
+            # self.evening_sounding = self.df_soundings.loc[self.morning_sounding['eval0']]
+
+            label = 'air_ap:theta'
+            axes[label].clear()
+
+            tbox['datetime'].set_text(\
+                self.frames['profiles']['record_yaml_ini'].pars.datetime.strftime("%Y/%m/%d %H:%M"))
+                # +\
+                # ' -> '+ \
+                # self.frames['profiles']['record_yaml_end_obs'].pars.datetime.strftime("%Y/%m/%d %H:%M"))
+            
+            
+            
+            
+            #+self.evening_sounding.datetime.strftime("%Y/%m/%d %H:%M")+ "UTC")
+            # 
+            #print('r12')
+
+            # #axes[label].set_title(self.morning_sounding.ldatetime.strftime("local time:  %H:%M")+' -> '+self.evening_sounding.ldatetime.strftime("%H:%M"))
+            # #axes[label].set_title(self.morning_sounding.datetime.strftime("%Y/%m/%d %H:%M") + ' -> '+self.evening_sounding.datetime.strftime("%Y/%m/%d %H:%M"))
+            # 
+            #print(self.frames['profiles']['record_yaml_ini'].pars.h)
+            #print(self.frames['profiles']['record_yaml_end_obs'].pars.h)
+            #print(self.frames['profiles']['record_yaml_end_mod'].out['h'].values[-1])
+            hmax = np.nanmax([self.frames['profiles']['record_yaml_ini'].pars.h,\
+                           self.frames['profiles']['record_yaml_end_mod'].out.h[-1]])
+            if self.path_forcing is not None:
+                hmax = np.nanmax([hmax,self.frames['profiles']['record_yaml_end_obs'].pars.h])
+
+
+                zidxmax = int(np.where((self.frames['profiles']['record_yaml_ini'].air_balloon.z.values
+                                    < 2.*hmax))[0][-1])+2
+                zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_ini'].air_balloon.z.values)))
+                zco = range(zidxmax)
+
+                axes[label].plot(self.frames['profiles']['record_yaml_ini'].air_balloon.theta.values[zco], \
+                                 self.frames['profiles']['record_yaml_ini'].air_balloon.z.values[zco],"b*", \
+                                 label="obs "+\
+                                 self.frames['profiles']['record_yaml_ini'].pars.ldatetime.strftime("%H:%M")\
+                                 +'LT')
+            #print('r14')
+            zidxmax = int(np.where((self.frames['profiles']['record_yaml_ini'].air_ap.z.values
+                                < 2.*hmax))[0][-1])+2
+            zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_ini'].air_ap.z.values)))
+            zco = range(zidxmax)
+
+            axes[label].plot(self.frames['profiles']['record_yaml_ini'].air_ap.theta.values[zco], \
+                             self.frames['profiles']['record_yaml_ini'].air_ap.z.values[zco],"b:", \
+                             label="fit "+\
+                             self.frames['profiles']['record_yaml_ini'].pars.ldatetime.strftime("%H:%M")\
+                             +'LT')
+
+
+            #print('r15')
+            if self.path_forcing is not None:
+                zidxmax = int(np.where((self.frames['profiles']['record_yaml_end_obs'].air_balloon.z.values
+                                    < 2.*hmax))[0][-1])+2
+                zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_end_obs'].air_balloon.z.values)))
+                zco = range(zidxmax)
+
+                              
+                axes[label].plot(self.frames['profiles']['record_yaml_end_obs'].air_balloon.theta.values[zco], \
+                                 self.frames['profiles']['record_yaml_end_obs'].air_balloon.z.values[zco],"r*", \
+                                 label="obs "+\
+                                 self.frames['profiles']['record_yaml_end_obs'].pars.ldatetime.strftime("%H:%M")\
+                                 +'LT')
+
+                #print('r16')
+
+                zidxmax = int(np.where((self.frames['profiles']['record_yaml_end_obs'].air_ap.z.values < 2.*hmax))[0][-1])+2
+                zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_end_obs'].air_ap.z.values)))
+                zco = range(zidxmax)
+
+                axes[label].plot(self.frames['profiles']['record_yaml_end_obs'].air_ap.theta.values[zco], \
+                                 self.frames['profiles']['record_yaml_end_obs'].air_ap.z.values[zco],"r:", \
+                                 label="fit "+\
+                                 self.frames['profiles']['record_yaml_end_obs'].pars.ldatetime.strftime("%H:%M")\
+                                 +'LT')
+
+            #print('r17')
+            #print(self.frames['profiles']['record_yaml_end_mod'].air_ap.z)
+            #print(hmax)
+            valid_mod = len(self.frames['profiles']['record_yaml_end_mod'].air_ap.z)>= 4
+            if valid_mod:
+
+                zidxmax = int(np.where((self.frames['profiles']['record_yaml_end_mod'].air_ap.z.values < 2.*hmax))[0][-1])+2
+                zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_end_mod'].air_ap.z.values)))
+                zco = range(zidxmax)
+
+                axes[label].plot(self.frames['profiles']['record_yaml_end_mod'].air_ap.theta.values[zco], \
+                                 self.frames['profiles']['record_yaml_end_mod'].air_ap.z.values[zco],"r-", \
+                                 label="mod "+\
+                                 (self.frames['profiles']['record_yaml_ini'].pars.ldatetime
+                                 +dt.timedelta(seconds=self.frames['profiles']['record_yaml_ini'].pars.runtime)).strftime("%H:%M")\
+                                 +'LT')
+
+            #print('r18')
+            axes[label].legend(prop={'family':'monospace'},loc='upper left')
+            axes[label].set_ylabel('height [m]')
+            axes[label].set_xlabel('theta [K]')
+
+            label = 'air_ap:q'
+            axes[label].clear()
+
+            tbox['datetime'].set_text(\
+                 (self.frames['profiles']['record_yaml_ini'].pars.datetime_daylight+\
+                  dt.timedelta(seconds=self.frames['profiles']['record_yaml_ini'].pars.runtime)).strftime("%Y/%m/%d %H:%M")
+                )
+            
+            #+self.evening_sounding.datetime.strftime("%Y/%m/%d %H:%M")+ "UTC")
+            # 
+
+            #print('r19')
+            # #axes[label].set_title(self.morning_sounding.ldatetime.strftime("local time:  %H:%M")+' -> '+self.evening_sounding.ldatetime.strftime("%H:%M"))
+            # #axes[label].set_title(self.morning_sounding.datetime.strftime("%Y/%m/%d %H:%M") + ' -> '+self.evening_sounding.datetime.strftime("%Y/%m/%d %H:%M"))
+            # 
+            if valid_mod:
+                hmax = np.nanmax([self.frames['profiles']['record_yaml_ini'].pars.h,\
+                               self.frames['profiles']['record_yaml_end_mod'].out.h[-1]])
+            else:
+                hmax = self.frames['profiles']['record_yaml_ini'].pars.h
+
+            if self.path_forcing is not None:
+                hmax = np.nanmax([hmax,self.frames['profiles']['record_yaml_end_obs'].pars.h])
+            # 
+            #print('r20')
+
+                zidxmax = int(np.where((self.frames['profiles']['record_yaml_ini'].air_balloon.z.values < 2.*hmax))[0][-1])+2
+                zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_ini'].air_balloon.z.values)))
+                zco = range(zidxmax)
+
+                axes[label].plot(self.frames['profiles']['record_yaml_ini'].air_balloon.q.values[zco], \
+                                 self.frames['profiles']['record_yaml_ini'].air_balloon.z.values[zco],"b*", \
+                                 label="obs "+\
+                                 self.frames['profiles']['record_yaml_ini'].pars.ldatetime.strftime("%H:%M")\
+                                 +'LT')
+                #print('r21')
+
+
+            zidxmax = int(np.where((self.frames['profiles']['record_yaml_ini'].air_ap.z.values < 2.*hmax))[0][-1])+2
+            zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_ini'].air_ap.z.values)))
+            zco = range(zidxmax)
+
+            axes[label].plot(self.frames['profiles']['record_yaml_ini'].air_ap.q.values[zco], \
+                             self.frames['profiles']['record_yaml_ini'].air_ap.z.values[zco],"b:", \
+                             label="fit "+\
+                             self.frames['profiles']['record_yaml_ini'].pars.ldatetime.strftime("%H:%M")\
+                             +'LT')
+
+            if self.path_forcing is not None:
+                zidxmax = int(np.where((self.frames['profiles']['record_yaml_end_obs'].air_balloon.z.values < 2.*hmax))[0][-1])+2
+                zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_end_obs'].air_balloon.z.values)))
+                zco = range(zidxmax)
+
+
+                axes[label].plot(self.frames['profiles']['record_yaml_end_obs'].air_balloon.q.values[zco], \
+                                 self.frames['profiles']['record_yaml_end_obs'].air_balloon.z.values[zco],"r*", \
+                                 label="obs "+\
+                                 self.frames['profiles']['record_yaml_end_obs'].pars.ldatetime.strftime("%H:%M")\
+                                 +'LT')
+
+                zidxmax = int(np.where((self.frames['profiles']['record_yaml_end_obs'].air_ap.z.values < 2.*hmax))[0][-1])+2
+                zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_end_obs'].air_ap.z.values)))
+                zco = range(zidxmax)
+
+                #print('r23')
+                axes[label].plot(self.frames['profiles']['record_yaml_end_obs'].air_ap.q.values[zco], \
+                                 self.frames['profiles']['record_yaml_end_obs'].air_ap.z.values[zco],"r:", \
+                                 label="fit "+\
+                                 self.frames['profiles']['record_yaml_end_obs'].pars.ldatetime.strftime("%H:%M")\
+                                 +'LT')
+
+            #print('r24')
+            if valid_mod:
+                zidxmax = int(np.where((self.frames['profiles']['record_yaml_end_mod'].air_ap.z.values < 2.*hmax))[0][-1])+2
+                zidxmax = np.min((zidxmax,len(self.frames['profiles']['record_yaml_end_mod'].air_ap.z.values)))
+                zco = range(zidxmax)
+                axes[label].plot(self.frames['profiles']['record_yaml_end_mod'].air_ap.q.values[zco], \
+                                 self.frames['profiles']['record_yaml_end_mod'].air_ap.z.values[zco],"r-", \
+                                 label="fit ")#+\
+                             #self.frames['profiles']['record_yaml_end_mod'].pars.ldatetime.strftime("%H:%M")\
+                             #+'LT')
+            #print('r25')
+            #axes[label].legend()
+
+            #axes[label].legend(prop={'family':'monospace'},loc='upper left')
+            #axes[label].set_ylabel('height [m]')
+            axes[label].set_xlabel('q [kg/kg]')
+
+            # #axes[label].set_title(self.evening_sounding.datetime.strftime("%Y/%m/%d %H:%M"))
+            # zco =  self.evening_sounding.obs.z_pro < 2.*hmax
+            # axes[label].plot(self.evening_sounding.obs.theta_pro[zco], self.evening_sounding.obs.z_pro[zco],"r*",label="obs "+self.evening_sounding.ldatetime.strftime("%H:%M")+'LT')
+            # 
+            # zco =  self.evening_sounding.fit.z_pro < 2.*hmax
+            # axes[label].plot(self.evening_sounding.fit.theta_pro[zco], self.evening_sounding.fit.z_pro[zco],"r:",label="fit "+self.evening_sounding.ldatetime.strftime("%H:%M")+'LT')
+            # 
+            # zco = self.morning_sounding.c4gl.z_pro < 2.*hmax
+            # axes[label].plot(self.morning_sounding.c4gl.theta_pro[zco], self.morning_sounding.c4gl.z_pro[zco],"r-",label="mod "+self.evening_sounding.ldatetime.strftime("%H:%M")+'LT')
+
+            # #pl.subplots_adjust(right=0.6)
+
+            # label = 'q_pro'
+            # axes[label].clear()
+
+            # hmax = np.max([self.morning_sounding.c4gl.input.h,self.morning_sounding.c4gl.out.h[-1],self.evening_sounding.fit.h])
+            # 
+            # zco =  self.morning_sounding.obs.z_pro < 2.*hmax
+            # axes[label].plot(self.morning_sounding.obs.q_pro[zco], self.morning_sounding.obs.z_pro[zco],"b*",label="obs")
+            # 
+            # zco =  self.morning_sounding.c4gl.input.z_pro < 2.*hmax
+            # axes[label].plot(self.morning_sounding.c4gl.input.q_pro[zco], self.morning_sounding.c4gl.input.z_pro[zco ],"b:",label="fit")
+
+            # #self.ax5.set_title(self.evening_sounding.ldatetime.strftime("local time: %H:%M"))
+            # zco =  self.evening_sounding.obs.z_pro < 2.*hmax
+            # axes[label].plot(self.evening_sounding.obs.q_pro[zco], self.evening_sounding.obs.z_pro[zco],"r*",label="obs")
+            # 
+            # zco =  self.evening_sounding.fit.z_pro < 2.*hmax
+            # axes[label].plot(self.evening_sounding.fit.q_pro[zco], self.evening_sounding.fit.z_pro[zco],"r:",label="fit")
+            # 
+            # zco = self.morning_sounding.c4gl.z_pro < 2.*hmax
+            # axes[label].plot(self.morning_sounding.c4gl.q_pro[zco], self.morning_sounding.c4gl.z_pro[zco],"r-",label="mod")
+            # #pl.subplots_adjust(right=0.6)
+            # axes[label].set_xlabel('specific humidity [kg/kg]')
+ 
+
+            #print('r26')
+            time = self.frames['profiles']['record_yaml_end_mod'].out.time
+            for ilabel,label in enumerate(['h','theta','q']):
+                axes["out:"+label].clear()
+                axes["out:"+label].plot(time,self.frames['profiles']['record_yaml_end_mod'].out.__dict__[label],label=label)
+                axes["out:"+label].set_ylabel(label)
+                if ilabel == 2:
+                    axes["out:"+label].set_xlabel('local sun time [h]')
+                
+            #print('r27')
+            label = 'SEB'
+            axes[label].clear()
+            
+            axes[label].plot(time,self.frames['profiles']['record_yaml_end_mod'].out.Swin - self.frames['profiles']['record_yaml_end_mod'].out.Swout,label='Sw')
+            axes[label].plot(time,-self.frames['profiles']['record_yaml_end_mod'].out.H,label='H')
+            axes[label].plot(time,self.frames['profiles']['record_yaml_end_mod'].out.Lwin - self.frames['profiles']['record_yaml_end_mod'].out.Lwout,label='Lw')
+            axes[label].plot(time,-self.frames['profiles']['record_yaml_end_mod'].out.G,label='G')
+            axes[label].plot(time,-self.frames['profiles']['record_yaml_end_mod'].out.LE,label='LE')
+            axes[label].hlines(0.,*axes[label].get_xlim(),'k')
+            axes[label].set_ylabel('energy flux [$\mathrm{W/m^2}$]')
+            axes[label].set_xlabel('local sun time [$\mathrm{h}$]')
+                
+            #print('r28')
+            
+            axes[label].legend()
+            
+            #         for ax in self.fig_timeseries_axes:
+#             ax.clear()
+#         
+#         self.fig_timeseries_axes[0].plot(self.morning_sounding.c4gl.out.h,label='h')
+#         self.fig_timeseries_axes[1].plot(self.morning_sounding.c4gl.out.theta,label='theta')
+#         self.fig_timeseries_axes[2].plot(self.morning_sounding.c4gl.out.q,label='q')
+#         #print(self.morning_sounding.c4gl.out.Swin)
+#         self.fig_timeseries_axes[3].plot(self.morning_sounding.c4gl.out.Swin - self.morning_sounding.c4gl.out.Swout,label='Sw')
+#         self.fig_timeseries_axes[3].plot(-self.morning_sounding.c4gl.out.H,label='H')
+#         self.fig_timeseries_axes[3].plot(self.morning_sounding.c4gl.out.Lwin - self.morning_sounding.c4gl.out.Lwout,label='Lw')
+#         self.fig_timeseries_axes[3].plot(-self.morning_sounding.c4gl.out.G,label='G')
+#         self.fig_timeseries_axes[3].plot(-self.morning_sounding.c4gl.out.LE,label='LE')
+#         self.fig_timeseries_axes[3].hlines(0.,*self.fig_timeseries_axes[3].get_xlim(),'k')
+#         self.fig_timeseries_axes[3].legend()
+#         self.fig.canvas.draw()
+            
+
+
+
+
+
+
+        #self.ready()
+        #print('r29')
+        fig.canvas.draw()
+        #fig.show()
+
+        self.axes = axes
+        self.tbox = tbox
+        self.fig = fig
+
+    def on_pick(self,event):
+        #print("HELLO")
+        # this makes clear that the dataset is loading (set_profile_focus takes a long time to load!)
+        #self.axes['theta_pro'].clear()
+        #self.axes['q_pro'].clear()
+        
+
+        # workaround because I cannot track the axes label here. I need it because the behaviour of this function should depend on which axes we are.
+        # I can only track the label of the data points. So we make a definition that clarifies to which axes the select data points (having a 'key') belongs to
+        keys_to_axes = {}
+        for ikey,key in enumerate(self.frames['stats']['viewkeys']):
+            keys_to_axes['d'+self.frames['stats']['viewkeys'][ikey]+'dt'] = 'stats_d'+key+'dt'
+
+        keys_to_axes['worldmap_stations'] = 'worldmap_stations'
+        keys_to_axes['worldmap'] = 'worldmap'
+        
+        axes = self.axes
+        #nstatsview = self.nstatsview
+        #statsviewcmap = self.statsviewcmap
+        stations = self.frames['worldmap']['stations'].table
+
+
+        #print("p1")
+        current = event
+        artist = event.artist
+        
+        selkey = artist.get_label()
+        
+        #print(keys_to_axes)
+        
+        label = keys_to_axes[selkey]
+        #print("HELLO",selkey,label)
+
+        # # Get to know in which axes we are
+        # label = None
+        # for axeskey in axes.keys():
+        #     if event.inaxes == axes[axeskey]:
+        #         label = axeskey
+        #         
+
+        # cont, pos = None, None
+        
+        xmouse, ymouse = event.mouseevent.xdata, event.mouseevent.ydata
+        ind = event.ind
+        # x, y = artist.get_xdata(), artist.get_ydata() # for some reason this doesnt work yet :/
+        d = axes[label].collections[0]
+        #d.set_offset_position('data')
+        xy = d.get_offsets()
+        x, y =  xy[:,0],xy[:,1]
+        #axes[-1].plot(seltableoutput[key+'_obs']*3600.,seltableoutput[key+'_end_mod']*3600.,'ro', markersize=5, picker=5,label=key)
+
+        #print("p2")
+        if len(ind) > 0:
+            #print("p3")
+            pos = x[ind[0]], y[ind[0]]
+
+            #if label[:-1] == 'statsview':
+            #    #seltablestatsstdrel = self.seltablestatsstdrel
+            #    #seltablestatspct = self.seltablestatspct
+
+            #    #self.set_statsviewfocus('STNID' seltablestatsstdrel[(seltablestatsstdrel[selkey+'_ext'] == pos[0]) & (seltablestatsstdrel[selkey+'_end_mod'] == pos[1] )  ].STNID.iloc[0]
+            #    #self.set_statsviewfocus('DT'] = seltablestatsstdrel[(seltablestatsstdrel[selkey+'_ext'] == pos[0]) & (seltablestatsstdrel[selkey+'_end_mod'] == pos[1] )  ].DT.iloc[0]
+            #    
+            #    self.axes['worldmap'].focus['STNID'] = self.axes['statsview0'].focus['STNID']
+            #    self.set_profilefocus(STNID=self.axes['statsview0'].focus['STNID'],DT=self.axes['statsview0'].focus['DT'])
+            #    self.goto_datetime_worldmap(self.profilefocus['DT'],'after')
+            #    
+            #    self.refresh_plot_interface(only=['statsviews_lightupdate','worldmap','profiles'],statsnewdata=False)
+            #el
+            if (label == 'worldmap') or (label == 'worldmap_stations'):
+                self.hover_active = False
+                if (self.frames['worldmap']['STNID'] != self.frames['profiles']['STNID']):
+                # WE ALREADY HAVE the correct station from worldmap/stats because of the hovering!!
+                # so we just need to perform update_station
+                    self.update_station()
+            elif (label[:5] == 'stats'):
+
+                self.hover_active = False
+                if (self.frames['stats']['STNID'] !=
+                self.frames['profiles']['STNID']) or \
+                   (self.frames['stats']['current_record_chunk'] != 
+                    self.frames['profiles']['current_record_chunk']) or \
+                   (self.frames['stats']['current_record_index'] != 
+                    self.frames['profiles']['current_record_index']):
+
+
+
+                    for key in ['STNID','current_station','stations_iterator']: 
+                        self.frames['worldmap'][key] = self.frames['stats'][key] 
+
+                    for key in self.frames['stats'].keys():
+                        self.frames['profiles'][key] = self.frames['stats'][key]
+
+                    STNID = self.frames['profiles']['STNID']
+                    chunk = self.frames['profiles']['current_record_chunk']
+                    if 'current_station_file_ini' in self.frames['profiles'].keys():
+                        self.frames['profiles']['current_station_file_ini'].close()
+                    self.frames['profiles']['current_station_file_ini'] = \
+                        open(self.path_exp+'/'+format(STNID,"05d")+'_'+str(chunk)+'_ini.yaml','r')
+
+                    if 'current_station_file_end_mod' in self.frames['profiles'].keys():
+                        self.frames['profiles']['current_station_file_end_mod'].close()
+                    self.frames['profiles']['current_station_file_end_mod'] = \
+                        open(self.path_exp+'/'+format(STNID,"05d")+'_'+str(chunk)+'_end.yaml','r')
+                    if self.path_forcing is not None:
+                        if 'current_station_file_end_obs' in self.frames['profiles'].keys():
+                            self.frames['profiles']['current_station_file_end_obs'].close()
+                        self.frames['profiles']['current_station_file_end_obs'] = \
+                            open(self.path_forcing+'/'+format(STNID,"05d")+'_end.yaml','r')
+
+                    # go to hovered record of current station
+                    self.frames['profiles']['records_iterator'] = \
+                                    records_iterator(self.frames['profiles']['records_current_station_end_mod'])
+                    # ... and go to the record of the profile window (last one that
+                    # was picked by the user)
+                    found = False
+                    EOF = False
+                    while (not found) and (not EOF):
+                        try:
+                            (STNID,chunk,index),record = self.frames['profiles']['records_iterator'].__next__()
+                            #print("hello*")
+                            #print(self.frames['profiles']['current_record_index'])
+                            if (chunk == self.frames['profiles']['current_record_chunk']) and \
+                               (index == self.frames['profiles']['current_record_index']) and \
+                               (STNID == self.frames['profiles']['STNID']):
+                                #print('found!')
+                                found = True
+                        except StopIteration:
+                            EOF = True
+                    if found:
+                        self.frames['stats']['current_record_end_mod'] = record
+                        self.frames['stats']['current_record_chunk'] = chunk
+                        self.frames['stats']['current_record_index'] = index
+                    # # for the profiles we make a distinct record iterator, so that the
+                    # # stats iterator can move independently
+                    # self.frames['profiles']['records_iterator'] = \
+                    #                 records_iterator(self.frames['profiles']['records_current_station_end_mod'])
+                    # (self.frames['profiles']['STNID'] , \
+                    # self.frames['profiles']['current_record_index']) , \
+                    # self.frames['profiles']['current_record_end_mod'] = \
+                    #                 self.frames['profiles']['records_iterator'].__next__()
+
+
+                    # for the profiles we make a distinct record iterator, so that the
+                    # stats iterator can move independently
+
+                    self.update_record()
+
+
+
+    def on_plot_hover(self,event):
+        axes = self.axes
+        #print('h1')
+
+        # Get to know in which axes we are
+        label = None
+        for axeskey in axes.keys():
+            if event.inaxes == axes[axeskey]:
+                label = axeskey
+                
+        #print('h2')
+
+        cont, pos = None, None
+        #print (label)
+        
+        if label is not None:
+            if  ('data' in axes[label].__dict__.keys()) and \
+                (label in axes[label].data.keys()) and \
+                (axes[label].data[label] is not None):
+                
+                #print('h3')
+                cont, ind =  axes[label].data[label].contains(event)
+                selkey = axes[label].data[label].get_label()
+                if len(ind["ind"]) > 0:
+                    #print('h4')
+                    pos = axes[label].data[label].get_offsets()[ind["ind"][0]]
+                    #print('pos',pos,selkey)
+
+
+                    #if label[:-1] == 'statsview':
+                    #    seltablestatsstdrel = self.seltablestatsstdrel
+                    #    seltablestatspct = self.seltablestatspct
+
+                    #    self.set_statsviewfocus('STNID'] = seltablestatsstdrel[(seltablestatsstdrel[selkey+'_ext'] == pos[0]) & (seltablestatsstdrel[selkey+'_end_mod'] == pos[1] )  ].STNID.iloc[0]
+                    #    self.set_statsviewfocus('DT'] = seltablestatsstdrel[(seltablestatsstdrel[selkey+'_ext'] == pos[0]) & (seltablestatsstdrel[selkey+'_end_mod'] == pos[1] )  ].DT.iloc[0]
+                    #    self.axes['worldmap'].focus['STNID'] = self.axes['statsview0'].focus['STNID']
+                    #    #self.goto_datetime_worldmap(self.axes['statsview0'].focus['DT'],'after')
+                    #    self.hover_active = True
+                    #    
+                    #    self.refresh_plot_interface(only=['statsviews_lightupdate','worldmap_stations'])
+                    #    
+                    #el
+                    #print(label[:5])
+                    if (label[:5] == 'stats') or (label == 'times'):
+                        # records_end_mod = self.frames['stats']['records_current_station_end_mod'][selkey]
+                        # records_obs = self.frames['stats']['records_current_station_end_obs'][selkey]
+                        
+                        if self.path_forcing is not None:
+                            if label[:5] == 'stats':
+                                records_end_mod_stats = self.frames['stats']['records_all_stations_end_mod_stats']
+                                records_obs_stats = self.frames['stats']['records_all_stations_end_obs_stats']
+                                (self.frames['stats']['STNID'] ,
+                                 self.frames['stats']['current_record_chunk'], 
+                                 self.frames['stats']['current_record_index']) = \
+                                    records_end_mod_stats[(records_obs_stats[selkey] == pos[0]) & (records_end_mod_stats[selkey] == pos[1])].index[0]
+                            # elif label[:5] == 'stats':
+                            #     # records_end_mod_stats = self.frames['stats']['records_all_stations_end_mod_stats']
+                            #     records_obs_stats = self.frames['stats']['records_all_stations_end_obs_stats']
+                            #     records_datetimes = self.frames['stats']['records_all_stations_ini']
+                            #     (self.frames['stats']['STNID'] ,
+                            #      self.frames['stats']['current_record_chunk'], 
+                            #      self.frames['stats']['current_record_index']) = \
+                            #         records_end_mod_stats[(records_obs_stats[selkey] == pos[0]) & (records_end_mod_stats[selkey] == pos[1])].index[0]
+
+
+                        self.frames['stats']['stations_iterator'] = stations_iterator(self.frames['worldmap']['stations']) 
+                        
+                        # # TO TEST: should be removed, since it's is also done just below
+                        # self.frames['stats']['stations_iterator'] = \
+                        #     self.frames['worldmap']['stations_iterator'] 
+                
+                
+                        # self.goto_datetime_worldmap(
+                        #     self.frames['profiles']['current_record_obs'].datetime.to_pydatetime(),
+                        #     'after')
+
+
+                        # scrolling to the right station
+                        STNID,station = self.frames['stats']['stations_iterator'].__next__()
+                        EOF = False
+                        found = False
+                        while (not found and not EOF):
+                            if (STNID == self.frames['stats']['STNID']):
+                                   found = True 
+                            if not found:
+                                try:
+                                    STNID,station = self.frames['stats']['stations_iterator'].__next__()
+                                except (StopIteration):
+                                    EOF = True
+                        if found:
+                        #    self.frames['stats']['STNID'] = STNID
+                            self.frames['stats']['current_station'] =  station
+
+                        #STNID = self.frames['profiles']['current_record_index'].iloc[0].name[0]
+                        #index = self.frames['profiles']['current_record_index'].iloc[0].name[1]
+
+
+                        # generate index of the current station
+                        self.frames['stats']['records_current_station_index'] = \
+                            (self.frames['stats']['records_all_stations_index'].get_level_values('STNID')\
+                             == self.frames['stats']['STNID'])
+
+
+                        tab_suffixes = \
+                                ['_end_mod','_ini','_ini_pct']
+                        if self.path_forcing is not None:
+                            tab_suffixes += \
+                                ['_end_mod_stats','_end_obs','_end_obs_stats']
+                            
+                        for tab_suffix in tab_suffixes:
+                            self.frames['stats']['records_current_station'+tab_suffix] = \
+                                self.frames['stats']['records_all_stations'+tab_suffix].iloc[self.frames['stats']['records_current_station_index']]
+
+
+
+                        # go to hovered record of current station
+                        self.frames['stats']['records_iterator'] = \
+                                        records_iterator(self.frames['stats']['records_current_station_end_mod'])
+
+
+                        # ... and go to the record of the profile window (last one that
+                        # was picked by the user)
+                        found = False
+                        EOF = False
+                        while (not found) and (not EOF):
+                            try:
+                                (STNID,chunk,index),record = self.frames['stats']['records_iterator'].__next__()
+                                #print("hello*")
+                                #print(self.frames['profiles']['current_record_index'])
+                                if (index == self.frames['stats']['current_record_index']) and \
+                                   (chunk == self.frames['stats']['current_record_chunk']) and \
+                                   (STNID == self.frames['stats']['STNID']):
+                                    #print('found!')
+                                    found = True
+                            except StopIteration:
+                                EOF = True
+                        if found:
+                            #print('h5')
+                            self.frames['stats']['current_record_end_mod'] = record
+                            self.frames['stats']['current_record_chunk'] = chunk
+                            self.frames['stats']['current_record_index'] = index
+
+                        #print(self.frames['stats']['STNID'],self.frames['stats']['current_record_index'])
+                        tab_suffixes = \
+                                ['_ini','_ini_pct']
+                        if self.path_forcing is not None:
+                            tab_suffixes += \
+                                ['_end_mod_stats','_end_obs','_end_obs_stats']
+                        for tab_suffix in tab_suffixes:
+                            #print(tab_suffix)
+                            #print(self.frames['stats']['records_current_station'+tab_suffix])
+                            self.frames['stats']['current_record'+tab_suffix] =  \
+                                self.frames['stats']['records_current_station'+tab_suffix].loc[\
+                                      (self.frames['stats']['STNID'] , \
+                                       self.frames['stats']['current_record_chunk'] , \
+                                       self.frames['stats']['current_record_index'])]
+
+
+                        self.hover_active = True
+                        self.refresh_plot_interface(only=['stats_lightupdate','worldmap_stations','profiles'])
+                        # print('h13')
+                        # if 'time' in self.globaldata.datasets[key].page[key].dims:
+                        #     self.goto_datetime_worldmap(
+                        #         self.frames['profiles']['current_record_ini'].datetime.to_pydatetime(),
+                        #         'after')
+                        #     if "fig" in self.__dict__.keys():
+                        #         self.refresh_plot_interface(only=['stats_lightupdate',
+                        #                                           'worldmap',
+                        #                                           'profiles'])
+                        # else:
+                        #     if "fig" in self.__dict__.keys():
+                        #         self.refresh_plot_interface(only=['stats_lightupdate',
+                        #                                           'worldmap_stations',
+                        #                                           'profiles'])
+
+
+
+                    elif label in ['worldmap_stations','worldmap']:
+                        #print('h5')
+
+                        if (self.axes['worldmap'].lat is not None) and \
+                           (self.axes['worldmap'].lon is not None):
+
+
+                            #self.loading()
+                            self.fig.canvas.draw()
+                            self.fig.show()
+
+
+                            # get position of 
+                            latmap = round(pos[1]/len(self.axes['worldmap'].lat)*(self.axes['worldmap'].lat[-1] - \
+                                                                 self.axes['worldmap'].lat[0]) + \
+                                           self.axes['worldmap'].lat[0],4)
+                            lonmap = round(pos[0]/len(self.axes['worldmap'].lon)*(self.axes['worldmap'].lon[-1] - \
+                                                                 self.axes['worldmap'].lon[0]) + \
+                                           self.axes['worldmap'].lon[0],4)
+                        
+                            stations = self.frames['worldmap']['stations'].table
+                            #print('h7')
+                        
+                            #reset stations iterator:
+                            # if 'stations_iterator' in self.frames['worldmap'].keys():
+                            #     self.frames['worldmap']['stations_iterator'].close()
+                            #     del(self.frames['worldmap']['stations_iterator'])
+                            # if 'stations_iterator' in self.frames['stats'].keys():
+                            #     self.frames['stats']['stations_iterator'].close()
+                            #     del(self.frames['stats']['stations_iterator'])
+                            self.frames['worldmap']['stations_iterator'] =\
+                               stations_iterator(self.frames['worldmap']['stations'])
+                            STNID,station = self.frames['worldmap']['stations_iterator'].__next__()
+                            EOF = False
+                            found = False
+                            while (not found and not EOF):
+                                #print('h8',station.latitude,latmap)
+                                #print('h8',station.longitude,lonmap)
+                                if (round(station.latitude,3) == round(latmap,3)) and \
+                                    (round(station.longitude,3) == round(lonmap,3)):
+                                       found = True 
+                                if not found:
+                                    try:
+                                        STNID,station = self.frames['worldmap']['stations_iterator'].__next__()
+                                    except (StopIteration):
+                                        EOF = True
+                            if found:
+                                self.frames['worldmap']['STNID'] = STNID
+                                self.frames['worldmap']['current_station'] = \
+                                        station
+                        
+                            self.frames['stats']['stations_iterator'] = \
+                                self.frames['worldmap']['stations_iterator'] 
+                            #print('h8')
+                            # inherit station position for the stats frame...
+                            for key in self.frames['worldmap'].keys():
+                                self.frames['stats'][key] = self.frames['worldmap'][key]
+                                
+                            ## fetch records of current station...
+                            #self.frames['stats']['records_current_station_end_mod'] =\
+                            #   get_records_end_mod(pd.DataFrame([self.frames['stats']['current_station']]),self.path_exp)
+
+                            # ... and their indices
+                            self.frames['stats']['records_current_station_index'] = \
+                                    (self.frames['stats']['records_all_stations_index'].get_level_values('STNID')\
+                                     == \
+                                     self.frames['stats']['current_station'].name)
+
+                            tab_suffixes = \
+                                    ['_end_mod','_ini','_ini_pct']
+                            if self.path_forcing is not None:
+                                tab_suffixes += \
+                                    ['_end_mod_stats','_end_obs','_end_obs_stats']
+
+                            for tab_suffix in tab_suffixes:
+                                self.frames['stats']['records_current_station'+tab_suffix] = \
+                                    self.frames['stats']['records_all_stations'+tab_suffix].iloc[self.frames['stats']['records_current_station_index']]
+
+
+                            # ... create a record iterator ...
+                            #self.frames['stats']['records_iterator'].close()
+                            del(self.frames['stats']['records_iterator'])
+                            self.frames['stats']['records_iterator'] = \
+                                self.frames['stats']['records_current_station_end_mod'].iterrows()
+
+
+
+                        
+                            #print('h9')
+                            # ... and go to to the first record of the current station
+                            (self.frames['stats']['STNID'] , \
+                             self.frames['stats']['current_record_chunk'] , \
+                             self.frames['stats']['current_record_index']) , \
+                            self.frames['stats']['current_record_end_mod'] = \
+                                self.frames['stats']['records_iterator'].__next__()
+                        
+                            tab_suffixes = \
+                                    ['_ini','_ini_pct']
+                            if self.path_forcing is not None:
+                                tab_suffixes += \
+                                    ['_end_mod_stats','_end_obs','_end_obs_stats']
+
+                            for tab_suffix in tab_suffixes:
+                                self.frames['stats']['current_record'+tab_suffix] =  \
+                                    self.frames['stats']['records_current_station'+tab_suffix].loc[\
+                                          (self.frames['stats']['STNID'] , \
+                                           self.frames['stats']['current_record_chunk'] , \
+                                           self.frames['stats']['current_record_index'])]
+
+                            #print('h11')
+                            
+                            self.hover_active = True
+                            self.refresh_plot_interface(only=['stats_lightupdate','worldmap_stations','profiles'])
+                            #print('h13')
+
+                        
+
+            #if (stations is not None):
+            #    for iSTN,STN in stations.iterrows():
+            ##        x,y =self.gmap(STN['longitude'],STN['latitude'])
+            ##        self.gmap.plot(x,y, 'mo' if (self.STNID == STN['ID']) else 'ro' , markersize=1)
+            #        x,y = len(axes[label].lon)*(STN['longitude']- axes[label].lon[0])/(axes[label].lon[-1] - axes[label].lon[0])  ,len(lat)*(STN['latitude']- axes[label].lat[0])/(lat[-1] - axes[label].lat[0])
+            #        axes['worldmap'].plot(x,y, 'mo' if (axes['worldmap'].focus['STNID'] == STN['ID']) else 'ro' , markersize=2)
+
+        # self.fig.show()
+ 
+        # we are hovering on nothing, so we are going back to the position of
+        # the profile sounding
+        if pos is None:
+            if self.hover_active == True:
+                #print('h1*')
+                
+                #self.loading()
+                # to do: reset stations iterators
+
+                # get station and record index from the current profile
+                for key in ['STNID', 'current_station']:
+                    self.frames['stats'][key] = self.frames['profiles'][key]
+
+                self.frames['stats']['STNID'] = self.frames['profiles']['STNID']
+                self.frames['stats']['current_station'] = \
+                        self.frames['profiles']['current_station']
+                #print('h3a*')
+                self.frames['stats']['records_current_station_end_mod'] = \
+                        self.frames['profiles']['records_current_station_end_mod']
+                #print('h3b*')
+
+                # the next lines recreate the records iterator. Probably it's
+                # better to just copy the profile iterator and its position to
+                # the worldmap/stats 
+
+                # reset stations iterator...
+                #self.frames['stats']['records_iterator'].close()
+                del(self.frames['stats']['records_iterator'])
+                self.frames['stats']['records_iterator'] = \
+                    self.frames['stats']['records_current_station_end_mod'].iterrows()
+                #print('h4*')
+
+                # ... and go to the record of the profile window (last one that
+                # was picked by the user)
+                found = False
+                EOF = False
+                while (not found) and (not EOF):
+                    try:
+                        (STNID,chunk,index),record = self.frames['stats']['records_iterator'].__next__()
+                        #print("hello*")
+                        #print(self.frames['profiles']['current_record_index'])
+                        #print(self.frames['profiles']['STNID'])
+                        #print(STNID,index)
+                        if (index == self.frames['profiles']['current_record_index']) and \
+                            (chunk == self.frames['profiles']['current_record_chunk']) and \
+                            (STNID == self.frames['profiles']['STNID']):
+                            #print('found!')
+                            found = True
+                    except StopIteration:
+                        EOF = True
+                if found:
+                    #print('h5*')
+                    self.frames['stats']['current_record_end_mod'] = record
+                    self.frames['stats']['current_record_chunk'] = chunk
+                    self.frames['stats']['current_record_index'] = index
+
+                #print('h6*')
+
+
+
+                # # fetch records of current station...
+                # self.frames['stats']['records_current_station_end_mod'] =\
+                #    get_records_end_mod(pd.DataFrame([self.frames['stats']['current_station']]),self.path_exp)
+
+                # ... and their indices
+                self.frames['stats']['records_current_station_index'] = \
+                        (self.frames['stats']['records_all_stations_index'].get_level_values('STNID')\
+                         == \
+                         self.frames['stats']['current_station'].name)
+
+                
+                tab_suffixes = \
+                        ['_ini','_ini_pct']
+                if self.path_forcing is not None:
+                    tab_suffixes += \
+                        ['_end_mod_stats','_end_obs','_end_obs_stats']
+
+                for tab_suffix in tab_suffixes:
+                    self.frames['stats']['records_current_station'+tab_suffix] = \
+                        self.frames['stats']['records_all_stations'+tab_suffix].iloc[self.frames['stats']['records_current_station_index']]
+
+
+                for tab_suffix in tab_suffixes:
+                    self.frames['stats']['current_record'+tab_suffix] =  \
+                        self.frames['stats']['records_current_station'+tab_suffix].loc[\
+                              (self.frames['stats']['STNID'] , \
+                               self.frames['stats']['current_record_chunk'] , \
+                               self.frames['stats']['current_record_index'])]
+
+
+                # the next lines recreate the stations iterator. Probably it's
+                # better to just copy the profile iterator and its position to
+                # the worldmap/stats 
+                #print('h7*')
+
+                # reset the stations iterators
+                for framekey in ['stats','worldmap']:
+                    ##print(framekey)
+                    if 'stations_iterator' in self.frames[framekey]:
+                        #self.frames[framekey]['stations_iterator'].close()
+                        del(self.frames[framekey]['stations_iterator'])
+
+                self.frames['worldmap']['current_station'] = \
+                        self.frames['profiles']['current_station']
+
+                #recreate the stations iterator for the worldmap...
+                self.frames['worldmap']['stations_iterator'] = stations_iterator(self.frames['worldmap']['stations']) 
+
+                # ... and go the position of the profile
+                #print('h8*')
+                STNID,station = self.frames['worldmap']['stations_iterator'].__next__()
+                EOF = False
+                found = False
+                while (not found and not EOF):
+                    if STNID == self.frames['profiles']['STNID'] :
+                        found = True 
+                    if not found:
+                        try:
+                            STNID,station = self.frames['worldmap']['stations_iterator'].__next__()
+                        except (StopIteration):
+                            EOF = True
+                if found:
+                    self.frames['worldmap']['current_station'] = station
+                    self.frames['worldmap']['STNID'] = STNID
+                #print('h9*')
+                self.frames['stats']['stations_iterator'] = \
+                    self.frames['worldmap']['stations_iterator'] 
+
+                # the stats window now inherits the current station from the
+                # worldmap
+                for key in ['STNID','current_station','stations_iterator']: 
+                    self.frames['stats'][key] = self.frames['worldmap'][key] 
+                #print('h10*')
+
+                # # we now only need inherit station position and go to first record
+                # for key in self.frames['worldmap'].keys():
+                #     self.frames['stats'][key] = self.frames['worldmap'][key]
+
+                # self.frames['stats']['records_current_station'] =\
+                #     get_records(pd.DataFrame().append(self.frames['stats']['current_station']))
+
+                # #print(self.frames['stats']['records_current_station'])
+                # self.frames['stats']['records_iterator'] = \
+                #                 self.frames['stats']['records_current_station'].iterrows()
+                # (self.frames['stats']['STNID'] , \
+                # self.frames['stats']['current_record_index']) , \
+                # self.frames['stats']['current_record_end_mod'] = \
+                #                 self.frames['stats']['records_iterator'].__next__()
+                
+
+
+
+
+
+
+                #self.set_statsviewfocus('STNID', self.profilefocus['STNID'])
+                ##self.set_statsviewfocus('DT'], self.profilefocus['DT'])
+                #self.axes['worldmap'].focus['STNID'] = self.profilefocus['STNID']
+                ##self.goto_datetime_worldmap(self.profilefocus['DT'],'after')
+                self.hover_active = False
+                self.refresh_plot_interface(only=['stats_lightupdate','worldmap_stations'],statsnewdata=False)
+    # def loading(self):
+    #     self.tbox['loading'].set_text('Loading...')
+    #     self.fig.canvas.draw()
+    #     self.fig.show()
+    #     sleep(0.1)
+    # def ready(self):
+    #     self.tbox['loading'].set_text('Ready')
+    #     self.fig.canvas.draw()
+    #     self.fig.show()
+
+
+
diff --git a/model.py b/class4gl/model.py
similarity index 50%
rename from model.py
rename to class4gl/model.py
index bb01df7..155b619 100644
--- a/model.py
+++ b/class4gl/model.py
@@ -9,7 +9,7 @@
 # This file is part of CLASS
 # 
 # CLASS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU General Public License as published bygamma
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 # 
@@ -25,25 +25,186 @@
 import copy as cp
 import numpy as np
 import sys
+import warnings
+import pandas as pd
+from ribtol.ribtol_hw import zeta_hs2 , funcsche
+import logging
+#from SkewT.thermodynamics import Density
 #import ribtol
 
+grav = 9.81
 def esat(T):
     return 0.611e3 * np.exp(17.2694 * (T - 273.16) / (T - 35.86))
 
 def qsat(T,p):
     return 0.622 * esat(T) / p
 
+
+def ribtol(Rib, zsl, z0m, z0h): 
+    Rib = np.float64(Rib)
+    zsl = np.float64(zsl)
+    z0m = np.float64(z0m)
+    z0h = np.float64(z0h)
+    #print(Rib,zsl,z0m,z0h)
+    if(Rib > 0.):
+        L    = 1.
+        L0   = 2.
+    else:
+        L  = -1.
+        L0 = -2.
+    #print(Rib,zsl,z0m,z0h)
+    while (abs(L - L0) > 0.001):
+        L0      = L
+        fx      = Rib - zsl / L * (np.log(zsl / z0h) - psih(zsl / L) + psih(z0h / L)) / (np.log(zsl / z0m) - psim(zsl / L) + psim(z0m / L))**2.
+        Lstart  = L - 0.001*L
+        Lend    = L + 0.001*L
+        fxdif   = ( (- zsl / Lstart * (np.log(zsl / z0h) - psih(zsl / Lstart) + psih(z0h / Lstart)) / \
+                                      (np.log(zsl / z0m) - psim(zsl / Lstart) + psim(z0m / Lstart))**2.) \
+                  - (-zsl /  Lend   * (np.log(zsl / z0h) - psih(zsl / Lend  ) + psih(z0h / Lend  )) / \
+                                      (np.log(zsl / z0m) - psim(zsl / Lend  ) + psim(z0m / Lend  ))**2.) ) / (Lstart - Lend)
+        L       = L - fx / fxdif
+        #print(L,fx/fxdif)
+        if(abs(L) > 1e12):
+            break
+
+    return L
+  
+def psim(zeta):
+    if(zeta <= 0):
+        x     = (1. - 16. * zeta)**(0.25)
+        psim  = 3.14159265 / 2. - 2. * np.arctan(x) + np.log((1. + x)**2. * (1. + x**2.) / 8.)
+        #x     = (1. + 3.6 * abs(zeta) ** (2./3.)) ** (-0.5)
+        #psim = 3. * np.log( (1. + 1. / x) / 2.)
+    else:
+        psim  = -2./3. * (zeta - 5./0.35) * np.exp(-0.35 * zeta) - zeta - (10./3.) / 0.35
+    return psim
+  
+def psih(zeta):
+    if(zeta <= 0):
+        x     = (1. - 16. * zeta)**(0.25)
+        psih  = 2. * np.log( (1. + x*x) / 2.)
+        #x     = (1. + 7.9 * abs(zeta) ** (2./3.)) ** (-0.5)
+        #psih  = 3. * np.log( (1. + 1. / x) / 2.)
+    else:
+        psih  = -2./3. * (zeta - 5./0.35) * np.exp(-0.35 * zeta) - (1. + (2./3.) * zeta) ** (1.5) - (10./3.) / 0.35 + 1.
+    return psih
+ 
 class model:
-    def __init__(self, model_input):
-        # initialize the different components of the model
-        self.input = cp.deepcopy(model_input)
+    def __init__(self, model_input = None,debug_level=None):
+
+        """ set up logger (see: https://docs.python.org/2/howto/logging.html)
+        """
+
+        self.logger = logging.getLogger('model')
+        if debug_level is not None:
+            self.logger.setLevel(logging._nameToLevel[debug_level])
+
+        """ initialize the different components of the model """ 
+
+        if model_input is not None:
+            # class4gl style input
+            if 'pars' in model_input.__dict__.keys():
+
+                # we make a reference to the full input first, so we can dump it
+                # afterwards
+                self.input_c4gl = model_input
+
+                # we copy the regular parameters first. We keep the classical input
+                # format as self.input so that we don't have to change the entire
+                # model code.
+                self.input = cp.deepcopy(model_input.pars)
+
+                # we copy other sections we are interested in, such as profile
+                # data, and store it also under input
+
+                # I know we mess up a bit the structure of the class4gl_input, but
+                # we will make it clean again at the time of dumping data
+
+                # So here, we copy the profile data into self.input
+                # 1. Air circulation data 
+                if 'sw_ac' in self.input.__dict__.keys() \
+                   and self.input.__dict__['sw_ac']:
+                    self.input.__dict__['air_ac'] = model_input.__dict__['air_ac']
+                    #self.input.__dict__['air_ach'] = model_input.__dict__['air_ach']
+
+                    # correct pressure of levels according to surface pressure
+                    # error (so that interpolation is done in a consistent way)
+
+                    p_prev = self.input.sp
+                    p_corr_prev = self.input.Ps
+                    #p_corr_prev = self.input.Ps - self.input.sp
+                    air_ac_index = self.input.air_ac.index
+                    for irow,indexrow in list(enumerate(air_ac_index))[::-1]:
+                        p_corr =self.input.air_ac.p.iloc[indexrow]/p_prev*p_corr_prev
+                        p_prev = self.input.air_ac.p.iloc[indexrow]
+
+                        self.input.air_ac.p.iloc[indexrow] = p_corr
+                        p_corr_prev = p_corr
+                       
+                       # p_old = self.input.air_ac.p.iloc[indexrow]
+
+                       # p_new = self.input.air_ac.p.iloc[indexrow] + p_corr
+
+                       # p_corr_next = np.log(self.input.air_ac.p.iloc[air_ac_index[irow-1]]/\
+                       #                      self.input.air_ac.p.iloc[indexrow]*p_corr)
+                       # self.input.air_ac.p.iloc[irow]=p_new
+
+                       # self.input.air_ac.p.iloc[irow] =\
+                       #  self.input.air_ac.p.iloc[irow] + p_e
+                       # p_e = p_e -\
+                       # (self.input.air_ac.p.iloc[irow]+p_e)/\
+                       #  self.input.air_ac.p.iloc[irow] *\
+                       #  self.input.air_ac.delpdgrav.iloc[irow]*grav
+
+            #     if 'ts' in model_input.__dict__.keys():
+            #         self.input.__dict__['ts'] = model_input.__dict__['ts']
+                if 'ts' in model_input.__dict__.keys():
+                    self.input.__dict__['ts'] = model_input.__dict__['ts']
+
+
+                # 2. Air circulation data 
+                if 'sw_ap' in self.input.__dict__.keys() \
+                   and self.input.__dict__['sw_ap']:
+                    self.input.__dict__['air_ap'] = model_input.__dict__['air_ap']
+
+            # standard class input
+            else:
+                self.input = cp.deepcopy(model_input)
+
+    def load_yaml_dict(self,yaml_dict):
+        dictouttemp = pd.DataFrame()
+        for key,data in yaml_dict.items():
+            if key == 'pars':
+                for keydata,value in data.items():
+                    self.__dict__[keydata] = value
+            elif key in ['air_ap','air_balloon','air_ac','air_ach']:
+                self.__dict__[key] = pd.DataFrame(data)
+            #elif key == 'sources':
+            #    self.__dict__[key] = data
+            elif key == 'out':
+                # lets convert it to a list of dictionaries
+                dictouttemp = pd.DataFrame(data).to_dict('list')
+            else: 
+                 warnings.warn("Key '"+key+"' is be implemented.")
+            #     self.__dict__[key] = data
+
+
+        if len(dictouttemp) > 0:
+            self.tsteps = len(dictouttemp['h'])
+            self.out = model_output(self.tsteps)
+            for keydictouttemp in dictouttemp.keys():
+                self.out.__dict__[keydictouttemp] = np.array(dictouttemp[keydictouttemp])
+
+
   
     def run(self):
         # initialize model variables
         self.init()
   
         # time integrate model 
-        for self.t in range(self.tsteps):
+        #for self.t in range(self.tsteps):
+        while self.t < self.tsteps:
+            print(self.t, '/', self.tsteps)
           
             # time integrate components
             self.timestep()
@@ -52,6 +213,7 @@ def run(self):
         self.exitmodel()
     
     def init(self):
+        self.dtmax = +np.inf
         # assign variables from input data
         # initialize constants
         self.Lv         = 2.5e6                 # heat of vaporization [J kg-1]
@@ -93,6 +255,7 @@ def init(self):
         self.R10        =  0.23;                # respiration at 10 C [mg CO2 m-2 s-1]
         self.E0         =  53.3e3;              # activation energy [53.3 kJ kmol-1]
 
+
         # Read switches
         self.sw_ml      = self.input.sw_ml      # mixed-layer model switch
         self.sw_shearwe = self.input.sw_shearwe # shear growth ABL switch
@@ -103,20 +266,38 @@ def init(self):
         self.sw_ls      = self.input.sw_ls      # land surface switch
         self.ls_type    = self.input.ls_type    # land surface paramaterization (js or ags)
         self.sw_cu      = self.input.sw_cu      # cumulus parameterization switch
+
+        self.sw_lit   = self.input.sw_lit       # switch for iterative L calculation
+        self.sw_ac    = self.input.sw_ac        # switch to take account of large-scale gridded Air Circulation (advection and subsidence) fields as input., eg., from ERA-INTERIM 
+        self.sw_ap    = self.input.sw_ap        # switch that tells to initialize with fitted Air Profiles (eg., from balloon soundings) as input
   
         # initialize mixed-layer
         self.h          = self.input.h          # initial ABL height [m]
         self.Ps         = self.input.Ps         # surface pressure [Pa]
+        self.sp         = self.input.sp         # This is also surface pressure
+                                                #but derived from the global data [Pa]
         self.divU       = self.input.divU       # horizontal large-scale divergence of wind [s-1]
         self.ws         = None                  # large-scale vertical velocity [m s-1]
         self.wf         = None                  # mixed-layer growth due to radiative divergence [m s-1]
-        self.fc         = self.input.fc         # coriolis parameter [s-1]
         self.we         = -1.                   # entrainment velocity [m s-1]
        
          # Temperature 
         self.theta      = self.input.theta      # initial mixed-layer potential temperature [K]
+        
+        
+        self.substep    = False
+        self.substeps   = 0
+
+
+
         self.dtheta     = self.input.dtheta     # initial temperature jump at h [K]
         self.gammatheta = self.input.gammatheta # free atmosphere potential temperature lapse rate [K m-1]
+
+        if 'gammatheta_lower_limit' not in self.input.__dict__.keys():
+            self.gammatheta_lower_limit = 0.002
+        else:
+            self.gammatheta_lower_limit = \
+                self.input.gammatheta_lower_limit # free atmosphere potential temperature lapse rate lower limit to avoid crashes [K m-1]
         self.advtheta   = self.input.advtheta   # advection of heat [K s-1]
         self.beta       = self.input.beta       # entrainment ratio for virtual heat [-]
         self.wtheta     = self.input.wtheta     # surface kinematic heat flux [K m s-1]
@@ -152,8 +333,14 @@ def init(self):
         self.wthetav    = None                  # surface kinematic virtual heat flux [K m s-1]
         self.wthetave   = None                  # entrainment kinematic virtual heat flux [K m s-1]
        
+        
+        
+        
+        
+        
         # Moisture 
         self.q          = self.input.q          # initial mixed-layer specific humidity [kg kg-1]
+
         self.dq         = self.input.dq         # initial specific humidity jump at h [kg kg-1]
         self.gammaq     = self.input.gammaq     # free atmosphere specific humidity lapse rate [kg kg-1 m-1]
         self.advq       = self.input.advq       # advection of moisture [kg kg-1 s-1]
@@ -167,6 +354,8 @@ def init(self):
         self.qsatsurf   = None                  # surface saturated specific humidity [g kg-1]
         self.dqsatdT    = None                  # slope saturated specific humidity curve [g kg-1 K-1]
       
+        
+        
         # CO2
         fac = self.mair / (self.rho*self.mco2)  # Conversion factor mgC m-2 s-1 to ppm m s-1
         self.CO2        = self.input.CO2        # initial mixed-layer CO2 [ppm]
@@ -189,7 +378,283 @@ def init(self):
         self.dv         = self.input.dv         # initial u-wind jump at h [m s-1]
         self.gammav     = self.input.gammav     # free atmosphere v-wind speed lapse rate [s-1]
         self.advv       = self.input.advv       # advection of v-wind [m s-2]
- 
+         
+  # BEGIN -- HW 20170606
+        # z-coordinate for vertical profiles of stratification above the mixed-layer height
+
+        if self.sw_ac:
+        # this is the data frame with the grided profile on the L60 grid
+        # (subsidence, and advection) 
+            self.air_ac      = self.input.air_ac  # full level air circulation
+                                                  # forcing
+            # self.air_ach     = self.input.air_ach # half level air circulation
+            #                                       # forcing
+            
+
+        if self.sw_ap:
+        # this is the data frame with the fitted profile (including HAGL,
+        # THTA,WSPD, SNDU,WNDV PRES ...)
+            self.air_ap      = self.input.air_ap  # initial profile of potential temperature [K]
+
+            # just for legacy reasons...
+            if 'z' not in list(self.air_ap.columns):
+                self.air_ap = self.air_ap.assign(z= lambda x: x.HAGL)
+            if 'p' not in list(self.air_ap.columns):
+                self.air_ap = self.air_ap.assign(p= lambda x: x.PRES*100.)
+
+            indexh = np.where(self.air_ap.z.values == self.h)
+            if (len(indexh) == 0) or (indexh[0][0] !=1) or (indexh[0][1] !=2):
+                raise ValueError("Error input profile consistency: mixed- \
+                                 layer height needs to be equal to the second \
+                                 and third \
+                                 level of the vertical profile input!")
+
+            # # initialize q from its profile when available
+            # p_old = self.Ps
+            # p_new = self.air_ap.p[indexh[0][0]]
+            # print(indexh)
+            # #stop
+            # 
+            # if ((p_old is not None) & (p_old != p_new)):
+            #     print("Warning: Ps input was provided ("+str(p_old)+\
+            #         "Pa), but it is now overwritten by the first level (index 0) of p_pro which is different ("\
+            #         +str(p_new)+"Pa).")
+            #                         
+            # self.Ps = p_new
+
+
+
+
+            # these variables/namings are more convenient to work with in the code
+            # we will update the original variables afterwards
+            #self.air_ap['q'] = self.air_ap.QABS/1000.
+
+            #work around for corrupt input
+            if 'level_0' in self.air_ap.columns:
+                self.air_ap = self.air_ap.drop(columns=['level_0'])
+
+            self.air_ap = \
+                    self.air_ap.assign(R= lambda x: self.Rd*(1.-x.q) + self.Rv*x.q)
+            # we require the temperature fields, since we need to consider
+            # advection
+            # if self.sw_ac:
+            #     #self.air_ap['theta'] = self.air_ap['t'] *
+
+            #     # we consider self.sp in case of air-circulation input (for
+            #     # consistence)
+            #     self.air_ap['t'] = \
+            #                 self.air_ap.theta *  \
+            #                 (self.air_ap.p/self.sp)**(self.air_ap['R']/self.cp)
+            # else:
+            # we consider self.Ps in case of balloon input only 
+            self.air_ap = self.air_ap.assign(t = lambda x: \
+                               x.theta * (x.p/self.Ps)**(x.R/self.cp))
+
+            #self.air_ap['theta'] = self.air_ap.THTA
+            if 'u' not in list(self.air_ap.columns):
+                self.air_ap = self.air_ap.assign(u = lambda x: x.WNDU)
+            if 'v' not in list(self.air_ap.columns):
+                self.air_ap = self.air_ap.assign(v = lambda x: x.WNDV)
+
+            for var in ['theta','q','u','v']:
+
+                
+                if self.air_ap[var][1] != self.air_ap[var][0]:
+                    raise ValueError("Error input profile consistency: two lowest profile levels for "+var+" should be equal.")
+                
+                # initialize the value from its profile when available
+                value_old = self.__dict__[var]
+                value_new = self.air_ap[var][indexh[0][0]]
+                
+                if ((value_old is not None) & (value_old != value_new)):
+                    warnings.warn("Warning:  input was provided ("+str(value_old)+ "), but it is now overwritten by the first level (index 0) of air_ap.var which is different (" +str(value_new)+").") 
+                self.__dict__[var] = value_new
+
+                # make a profile of the stratification 
+                # please note that the stratification between z_pro[i] and
+                # z_pro[i+1] is given by air_ap.GTHT[i]
+
+                # self.air_ap.GTHT = np.gradient(self.air_ap.THTA) /
+                # np.gradient(self.z_pro)
+                with np.errstate(divide='ignore'):
+                    gammavar = list(np.array(self.air_ap[var][1:].values - \
+                                             self.air_ap[var][:-1].values) \
+                                    / np.array(self.air_ap['z'][1:].values - \
+                                               self.air_ap['z'][:-1].values))
+
+                # add last element twice (since we have one element less)
+                gammavar.append(gammavar[-1])
+                gammavar = np.array(gammavar)
+                self.air_ap = self.air_ap.assign(**{'gamma'+var : gammavar})
+
+                value_old = self.__dict__['d'+var]
+                value_new = self.air_ap[var][2] - self.air_ap[var][1]
+                if ((value_old is not None) & (value_old != value_new)):
+                    warnings.warn("Warning:  input was provided ("+str(value_old)+ "), but it is now overwritten by the first level (index 0) of air_ap.dvar which is different (" +str(value_new)+").") 
+                self.__dict__['d'+var] = value_new
+
+                # gammatheta, gammaq, gammau, gammav are updated here.
+                self.__dict__['gamma'+var] = \
+                    self.air_ap['gamma'+var][np.where(self.h >= \
+                                                     self.air_ap.z)[0][-1]]
+
+
+
+        # the variable p_pro is just for diagnosis of lifted index
+            
+            
+
+            # input Ph is wrong, so we correct it according to hydrostatic equation
+            #self.Ph = self.Ps - self.h * self.g * Density(self.T2m,self.Ps,self.q)
+
+            #if self.sw_ac:
+                # note that we use sp as surface pressure, which is determined
+                # from era-interim instead of the observations. This is to
+                # avoid possible failure of the interpolation routine
+                # self.air_ap.p = np.array([self.Ps, self.P_h, self.P_h-0.1]\
+                #                          + \
+                #                          list(self.air_ap.p[3:]))
+
+            # else:
+                # in the other case, it is updated at the time of calculting
+                # the statistics 
+
+# END -- HW 20170606      
+        #print(self.air_ap)
+
+        if self.sw_ac and not self.sw_ap:
+            raise ValueError("air circulation switch only possible when air \
+                             profiles are given")
+        
+        if self.sw_ac:
+
+            # # # we comment this out, because subsidence is calculated
+            # according to advection
+            # #interpolate subsidence towards the air_ap height coordinate
+            # self.air_ap['w'] = np.interp(self.air_ap.p,\
+            #                               self.air_ac.p,\
+            #                               self.air_ac.w) 
+            # #subsidence at the mixed-layer top
+            # self.w = self.air_ap.w[1]
+        
+            self.P_h    = self.Ps - self.rho * self.g * self.h
+            in_ml = (self.air_ac.p >= self.P_h)
+
+            if (self.sw_ac is not None) and ('adv' in self.sw_ac):
+                # in case we didn't find any points, we just take the lowest one.
+                # actually, this can happen if ERA-INTERIM pressure levels are
+                # inconsistent with 
+                if in_ml.sum() == 0:
+                    warnings.warn(" no circulation points in the mixed layer \
+                                  found. We just take the bottom one.")
+                    in_ml = self.air_ac.index == (len(self.air_ac) - 1)
+
+                for var in ['theta','q','u','v']:
+    
+                   # calculation of the advection variables for the mixed layer
+                   # we weight by the hydrostatic thickness of each layer and
+                   # divide by the total thickness
+                   if var == 'theta':
+                       self.__dict__['adv'+var] = \
+                                ((self.air_ac['advt_x'][in_ml] \
+                                 + \
+                                 self.air_ac['advt_y'][in_ml])* \
+                                self.air_ac['delpdgrav'][in_ml]).sum()/ \
+                                self.air_ac['delpdgrav'][in_ml].sum()
+                   else:
+                       self.__dict__['adv'+var] = \
+                                ((self.air_ac['adv'+var+'_x'][in_ml] \
+                                 + \
+                                 self.air_ac['adv'+var+'_y'][in_ml])* \
+                                self.air_ac['delpdgrav'][in_ml]).sum()/ \
+                                self.air_ac['delpdgrav'][in_ml].sum()
+
+                   # calculation of the advection variables for the profile above
+                   # (lowest 3 values are not used by class)
+                   self.air_ap = self.air_ap.assign(**{'adv'+var : 0.})
+
+                   if var == 'theta':
+                       self.air_ap['adv'+var] = \
+                               np.interp(self.air_ap.p,\
+                                         self.air_ac.p,\
+                                         self.air_ac['advt_x']) \
+                               + \
+                               np.interp(self.air_ap.p, \
+                                           self.air_ac.p, \
+                                           self.air_ac['advt_y'])
+                   else:
+                       self.air_ap['adv'+var] = \
+                               np.interp(self.air_ap.p,\
+                                         self.air_ac.p,\
+                                         self.air_ac['adv'+var+'_x']) \
+                               + \
+                               np.interp(self.air_ap.p, \
+                                           self.air_ac.p, \
+                                           self.air_ac['adv'+var+'_y'])
+
+                # as an approximation, we consider that advection of theta in the
+                # mixed layer is equal to advection of t. This is a sufficient
+                # approximation since theta and t are very similar at the surface
+                # pressure.
+                # self.__dict__['advtheta'] = self.__dict__['advt']
+
+
+            # # # STRANGE, THIS DOESN'T GIVE REALISTIC VALUES, IT NEEDS TO BE
+            # # # CHECKED AGAIN SINCE THERE IS SIMILAR STRATEGY USED FOR 
+            # # # CALCULATING THE ADVECTION PROFILES
+            # # interpolate subsidence x density
+            # self.air_ap['wrho'] = \
+            #            np.interp(self.air_ap.p,\
+            #                      self.air_ach.p,\
+            #                      self.air_ach['wrho']) \
+            #     
+            # self.air_ap['w'] = \
+            #     self.air_ap['wrho']/(self.air_ap.p/ \
+            #                          (self.Rd*(1.-self.air_ap.q) + \
+            #                           self.Rv*self.air_ap.q)* \
+            #                          self.air_ap.TEMP)
+            # self.wrho = np.interp(self.P_h,\
+            #                      self.air_ach.p,\
+            #                      self.air_ach['wrho']) 
+            # self.ws   = self.air_ap.w.iloc[1]
+
+            if (self.sw_ac is not None) and \
+               (('w' in self.sw_ac) or ('adv' in self.sw_ac)):
+                self.air_ap = self.air_ap.assign(R = 0.)
+                self.air_ap['R'] = (self.Rd*(1.-self.air_ap.q) + \
+                                                     self.Rv*self.air_ap.q)
+                self.air_ap = self.air_ap.assign(rho = 0.)
+                self.air_ap['t'] = \
+                            self.air_ap.theta * \
+                            (self.air_ap.p/self.Ps)**(self.air_ap['R']/self.cp)
+                self.air_ap['rho'] = self.air_ap.p /self.air_ap.R/  self.air_ap.t
+
+            if (self.sw_ac is not None) and ('w' in self.sw_ac):
+                self.air_ap = self.air_ap.assign(wp = 0.)
+                self.air_ap['wp'] = np.interp(self.air_ap.p, \
+                                              self.air_ac.p, \
+                                              self.air_ac['wp'])
+                
+                self.air_ap = self.air_ap.assign(w = 0.)
+                self.air_ap['w'] = -self.air_ap['wp'] /self.air_ap['rho']/self.g
+                #print('hello w ini')
+
+                # Note: in case of sw_ac is False, we update it from prescribed
+                # divergence
+                self.ws   = self.air_ap.w[1]
+
+                # self.ws   = self.wrho/self.rho
+                # self.ws   = self.wrho/(self.P_h/ \
+                #                        (self.Rd*(1.-self.q) + self.Rv*self.q) * \
+                #                         self.theta) # this should be T!!!
+
+                # self.__dict__['divU'] = ((self.air_ac['divU_x'][in_ml] \
+                #                         + \
+                #                         self.air_ac['divU_y'][in_ml])* \
+                #             self.air_ac['delpdgrav'][in_ml]).sum()/ \
+                #             self.air_ac['delpdgrav'][in_ml].sum() \
+        
+
         # Tendencies 
         self.htend      = None                  # tendency of CBL [m s-1]
         self.thetatend  = None                  # tendency of mixed-layer potential temperature [K s-1]
@@ -218,11 +683,14 @@ def init(self):
   
         # initialize radiation
         self.lat        = self.input.lat        # latitude [deg]
+        #self.fc         = self.input.fc         # coriolis parameter [s-1]
+        self.fc         = 4. * np.pi/(24.*3600.) * np.sin(self.lat/180.*np.pi)
         self.lon        = self.input.lon        # longitude [deg]
         self.doy        = self.input.doy        # day of the year [-]
         self.tstart     = self.input.tstart     # time of the day [-]
         self.cc         = self.input.cc         # cloud cover fraction [-]
         self.Swin       = None                  # incoming short wave radiation [W m-2]
+        self.Swin_cs    = None                  # incoming short wave radiation clearsky [W m-2]
         self.Swout      = None                  # outgoing short wave radiation [W m-2]
         self.Lwin       = None                  # incoming long wave radiation [W m-2]
         self.Lwout      = None                  # outgoing long wave radiation [W m-2]
@@ -282,7 +750,7 @@ def init(self):
 
         # initialize A-Gs surface scheme
         self.c3c4       = self.input.c3c4       # plant type ('c3' or 'c4')
- 
+
         # initialize cumulus parameterization
         self.sw_cu      = self.input.sw_cu      # Cumulus parameterization switch
         self.dz_h       = self.input.dz_h       # Transition layer thickness [m]
@@ -293,6 +761,8 @@ def init(self):
         # initialize time variables
         self.tsteps = int(np.floor(self.input.runtime / self.input.dt))
         self.dt     = self.input.dt
+        self.dtcur      = self.dt
+        self.firsttime = True
         self.t      = 0
  
         # Some sanity checks for valid input
@@ -324,47 +794,103 @@ def init(self):
             self.run_mixed_layer()
 
     def timestep(self):
+        self.dtmax = +np.inf
+
+        self.logger.debug('before stats') 
         self.statistics()
 
         # run radiation model
+        self.logger.debug('before rad') 
         if(self.sw_rad):
             self.run_radiation()
   
         # run surface layer model
         if(self.sw_sl):
+            self.logger.debug('before surface layer') 
             self.run_surface_layer()
         
         # run land surface model
         if(self.sw_ls):
+            self.logger.debug('before land surface') 
             self.run_land_surface()
  
         # run cumulus parameterization
         if(self.sw_cu):
+            self.logger.debug('before cumulus') 
             self.run_cumulus()
    
+        self.logger.debug('before mixed layer') 
         # run mixed-layer model
         if(self.sw_ml):
             self.run_mixed_layer()
+        self.logger.debug('after mixed layer') 
  
+        #get first profile data point above mixed layer
+        if self.sw_ap:
+            zidx_first = np.where(self.air_ap.z > self.h)[0][0]
+            
+            if (self.sw_ac is not None) and ('w' in self.sw_ac):
+                # here we correct for the fact that the upper profile also
+                # shifts in the vertical.
+
+                diffhtend = self.htend - self.air_ap.w[zidx_first]
+                if diffhtend > 0:
+                    dtmax_new = (self.air_ap.z[zidx_first] - self.h)/ diffhtend
+                    self.dtmax= min(dtmax_new,self.dtmax)
+            else:
+                if self.htend > 0:
+                    dtmax_new = ( self.air_ap.z[zidx_first] - self.h)/self.htend 
+                    self.dtmax= min(dtmax_new,self.dtmax)
+            #print(self.h,zidx_first,self.ws,self.air_ap.z)
+
+        
+        #print(self.t,self.dtcur,self.dt,dtmax,self.air_ap.z[zidx_first],self.h)
+        self.logger.debug('before store') 
+        self.substep =  (self.dtcur > self.dtmax)
+        if self.substep:
+            dtnext = self.dtcur - self.dtmax
+            self.dtcur = self.dtmax
+
+        #print(self.t,self.dtcur,self.dt,dtmax,self.tstart + self.t*self.dt/3600.)
+
+        # HW: this will be done multiple times in case of a substep is needed
         # store output before time integration
-        self.store()
+        if self.firsttime:
+            self.store()
   
+        self.logger.debug('before integrate land surface ('+str(self.t)+', '+str(self.dtcur)+')')
         # time integrate land surface model
         if(self.sw_ls):
             self.integrate_land_surface()
-  
+        self.logger.debug('before integrate mixed layer') 
         # time integrate mixed-layer model
         if(self.sw_ml):
-            self.integrate_mixed_layer()
+            self.integrate_mixed_layer() 
+        self.logger.debug('after integrate mixed layer') 
+        if self.substep:
+            self.dtcur = dtnext
+            self.firsttime = False
+            self.substeps += 1
+        else:
+            self.dtcur = self.dt
+            self.t += 1 
+            self.firsttime = True
+            self.substeps = 0
+        self.logger.debug('going to next step')
+        
+        
   
     def statistics(self):
         # Calculate virtual temperatures 
         self.thetav   = self.theta  + 0.61 * self.theta * self.q
         self.wthetav  = self.wtheta + 0.61 * self.theta * self.wq
         self.dthetav  = (self.theta + self.dtheta) * (1. + 0.61 * (self.q + self.dq)) - self.theta * (1. + 0.61 * self.q)
-
         # Mixed-layer top properties
         self.P_h    = self.Ps - self.rho * self.g * self.h
+        # else:
+            # in the other case, it is updated at the time that the profile is
+            # updated (and at the initialization
+
         self.T_h    = self.theta - self.g/self.cp * self.h
 
         #self.P_h    = self.Ps / np.exp((self.g * self.h)/(self.Rd * self.theta))
@@ -389,8 +915,9 @@ def statistics(self):
             it          += 1
 
         if(it == itmax):
+
             print("LCL calculation not converged!!")
-            print("RHlcl = %f, zlcl=%f"%(RHlcl, self.lcl))
+            print("RHlcl = %f, zlcl=%f, theta=%f, q=%f"%(RHlcl, self.lcl,self.theta,self.q))
 
     def run_cumulus(self):
         # Calculate mixed-layer top relative humidity variance (Neggers et. al 2006/7)
@@ -417,10 +944,17 @@ def run_mixed_layer(self):
             # decompose ustar along the wind components
             self.uw = - np.sign(self.u) * (self.ustar ** 4. / (self.v ** 2. / self.u ** 2. + 1.)) ** (0.5)
             self.vw = - np.sign(self.v) * (self.ustar ** 4. / (self.u ** 2. / self.v ** 2. + 1.)) ** (0.5)
-      
+
+
+
         # calculate large-scale vertical velocity (subsidence)
-        self.ws = -self.divU * self.h
-      
+        if not ((self.sw_ac is not None) and ('w' in self.sw_ac)):
+            self.ws = -self.divU * self.h
+        # else:
+        #     in case the air circulation switch is turned on, subsidence is
+        #     calculated from the circulate profile at the initialization and
+        #     in the integrate_mixed_layer routine
+              
         # calculate compensation to fix the free troposphere in case of subsidence 
         if(self.sw_fixft):
             w_th_ft  = self.gammatheta * self.ws
@@ -448,39 +982,127 @@ def run_mixed_layer(self):
             self.we    = (-self.wthetave + 5. * self.ustar ** 3. * self.thetav / (self.g * self.h)) / self.dthetav
         else:
             self.we    = -self.wthetave / self.dthetav
-
         # Don't allow boundary layer shrinking if wtheta < 0 
         if(self.we < 0):
             self.we = 0.
 
         # Calculate entrainment fluxes
         self.wthetae     = -self.we * self.dtheta
+
+        #ent2
+        #self.wthetae         = self.input.ts.wthetae[self.t] #-self.we * self.dq
+
         self.wqe         = -self.we * self.dq
+        #ent and ent2  
+        #self.wqe         = self.input.ts.wqe[self.t] #-self.we * self.dq
         self.wCO2e       = -self.we * self.dCO2
-  
-        self.htend       = self.we + self.ws + self.wf - self.M
-       
-        self.thetatend   = (self.wtheta - self.wthetae             ) / self.h + self.advtheta 
-        self.qtend       = (self.wq     - self.wqe     - self.wqM  ) / self.h + self.advq
-        self.CO2tend     = (self.wCO2   - self.wCO2e   - self.wCO2M) / self.h + self.advCO2
+
+        # none(bot entrianment on) - ent2 (only dry air entrainment off): dry air entrainment effect
+        # ent2(only dry air entrainment off )  - ent (both theta and q entrainment off): dry air entrainment effect
+        # none - ent: full entrainment
+
+
+        
+        htend_pre       = self.we + self.ws + self.wf - self.M
+        
+        #self.thetatend   = (self.wtheta - self.wthetae             ) / self.h + self.advtheta 
+        thetatend_pre = (self.wtheta - self.wthetae             ) / self.h + self.advtheta
+        
+ 
+        #print('thetatend_pre',thetatend_pre)
         
-        self.dthetatend  = self.gammatheta * (self.we + self.wf - self.M) - self.thetatend + w_th_ft
-        self.dqtend      = self.gammaq     * (self.we + self.wf - self.M) - self.qtend     + w_q_ft
-        self.dCO2tend    = self.gammaCO2   * (self.we + self.wf - self.M) - self.CO2tend   + w_CO2_ft
+        #preliminary boundary-layer top chenage
+        #htend_pre = self.we + self.ws + self.wf - self.M
+        #preliminary change in temperature jump
+        dthetatend_pre  = self.gammatheta * (self.we + self.wf - self.M) - \
+                          thetatend_pre + w_th_ft
+        
+        dtheta_pre = float(self.dtheta + dthetatend_pre *self.dt)
+        self.l_entrainment = True
+
+        if (self.dtheta <= 0.1) and (dthetatend_pre < 0.):
+            self.l_entrainment = False
+            warnings.warn(str(self.t)+"/"+str(self.tsteps)+\
+                          " Warning! temperature jump is at the lower limit and is not growing: entrainment is disabled for this (sub)timestep.") 
+        elif dtheta_pre < 0.1:
+            dtmax_new = float((0.1 - self.dtheta)/dthetatend_pre)
+            self.l_entrainment = True
+            warnings.warn(str(self.t)+"/"+str(self.tsteps)+\
+                          " Warning! Potential temperature jump at mixed-layer height would become too low. So I'm limiting the timestep from "+ str(self.dtmax)+' to '+str(dtmax_new))
+            self.dtmax = min(self.dtmax,dtmax_new)
+            warnings.warn(str(self.t)+"/"+str(self.tsteps)+\
+                          " Next subtimestep, entrainment will also be disabled")
+            #self.dthetatend = (0.1 - self.dtheta)/self.dtcur 
+
+
+
+        # when entrainment is disabled, we just use the simplified formulation
+        # as in Wouters et al., 2013 (section 2.2.1)
+
+        self.dthetatend = self.l_entrainment*dthetatend_pre + \
+                        (1.-self.l_entrainment)*0.
+        self.thetatend = self.l_entrainment*thetatend_pre + \
+                        (1.-self.l_entrainment)*((self.wtheta  ) / self.h + self.advtheta)
+        self.htend = self.l_entrainment*htend_pre + \
+                     (1.-self.l_entrainment)*((self.ws - self.M)+ self.thetatend/self.gammatheta)
+        #print(l_entrainment,htend_pre,self.ws,self.M,self.thetatend,self.gammatheta)
+        #stop
+
+        # qtend_pre = (self.wq     - l_entrainment*self.wqe     - self.wqM  ) / self.h + self.advq
+        # q_pre = float(self.q + dtend_pre *self.dt)
+        # if q_pre < 0:
+        #     self.qtend = 
+
+        self.qtend       = (self.wq     - self.l_entrainment*self.wqe     - self.wqM  ) / self.h + self.advq
+        self.CO2tend     = (self.wCO2   - self.l_entrainment*self.wCO2e   - self.wCO2M) / self.h + self.advCO2
+
+
+        # self.qtend = l_entrainment*qtend_pre + \
+        #              (1.-l_entrainment)*( (self.wq  - self.wqM)/self.h + self.advq)
+        # self.CO2tend = l_entrainment*CO2tend_pre + \
+        #              (1.-l_entrainment)*( (self.wCO2  - self.wCO2M)/self.h + self.advCO2)
+
+
+
+        #     # part of the timestep for which the temperature mixed-layer jump
+        #     # was changing, and for which entrainment took place. For the other
+        #     # part, we don't assume entrainment anymore, and we use the
+        #     # simplified formulation  of Wouters et al., 2013
+
+        #     #self.htend =(self.dthetatend + self.thetatend - w_th_ft)/self.gammatheta +self.ws
+        #   
+        #     self.thetatend = l_entrainment*(self.gammatheta * (self.we + self.wf - self.M) - \
+        #                      self.dthetatend + w_th_ft) + \
+        #                      l_entrainment*((self.wtheta  ) / self.h + self.advtheta)
+        #     self.htend = fac*self.htend + \
+        #                  (1.-fac)* (( self.ws  - self.M)+((self.wtheta) / self.h + self.advtheta)/self.gammatheta)
+        #     self.qtend = fac*self.qtend + (1.-fac)* ( (self.wq  - self.wqM)/self.h + self.advq)
+        #     self.CO2tend = fac*self.qtend + (1.-fac)* ( (self.wCO2  - self.wCO2M)/self.h + self.advCO2)
+
+        #     #self.thetatend += (self.wtheta - self.wthetae             ) / self.h + self.advtheta
+
+        # else:
+        #     #self.htend = htend_pre
+        #     self.dthetatend = dthetatend_pre
+        #     self.thetatend = thetatend_pre
+        
+        self.dqtend      = self.gammaq     * (self.we*self.l_entrainment + self.wf - self.M) - self.qtend     + w_q_ft
+        self.dCO2tend    = self.gammaCO2   * (self.we*self.l_entrainment + self.wf - self.M) - self.CO2tend   + w_CO2_ft
      
         # assume u + du = ug, so ug - u = du
         if(self.sw_wind):
-            self.utend       = -self.fc * self.dv + (self.uw + self.we * self.du)  / self.h + self.advu
-            self.vtend       =  self.fc * self.du + (self.vw + self.we * self.dv)  / self.h + self.advv
+            self.utend       = -self.fc * self.dv + (self.uw + self.l_entrainment*self.we * self.du)  / self.h + self.advu
+            self.vtend       =  self.fc * self.du + (self.vw + self.l_entrainment*self.we * self.dv)  / self.h + self.advv
   
-            self.dutend      = self.gammau * (self.we + self.wf - self.M) - self.utend
-            self.dvtend      = self.gammav * (self.we + self.wf - self.M) - self.vtend
+            self.dutend      = self.gammau * (self.l_entrainment*self.we + self.wf - self.M) - self.utend
+            self.dvtend      = self.gammav * (self.l_entrainment*self.we + self.wf - self.M) - self.vtend
         
         # tendency of the transition layer thickness
         if(self.ac > 0 or self.lcl - self.h < 300):
             self.dztend = ((self.lcl - self.h)-self.dz_h) / 7200.
         else:
             self.dztend = 0.
+
    
     def integrate_mixed_layer(self):
         # set values previous time step
@@ -501,26 +1123,374 @@ def integrate_mixed_layer(self):
         dz0     = self.dz_h
   
         # integrate mixed-layer equations
-        self.h        = h0      + self.dt * self.htend
-        self.theta    = theta0  + self.dt * self.thetatend
-        self.dtheta   = dtheta0 + self.dt * self.dthetatend
-        self.q        = q0      + self.dt * self.qtend
-        self.dq       = dq0     + self.dt * self.dqtend
-        self.CO2      = CO20    + self.dt * self.CO2tend
-        self.dCO2     = dCO20   + self.dt * self.dCO2tend
-        self.dz_h     = dz0     + self.dt * self.dztend
+        
+            
+
+# END -- HW 20170606        
+        self.h        = h0      + self.dtcur * self.htend
+        # print(self.h,self.htend)
+        # stop
+        self.theta    = theta0  + self.dtcur * self.thetatend
+        #print(dtheta0,self.dtcur,self.dthetatend)
+        self.dtheta   = dtheta0 + self.dtcur * self.dthetatend
 
+
+        # qtend_pre = (self.wq     - l_entrainment*self.wqe     - self.wqM  ) / self.h + self.advq
+        # q_pre = float(self.q + dtend_pre *self.dt)
+        # if q_pre < 0:
+        #     self.qtend = 
+
+        # calculate compensation to fix the free troposphere in case of subsidence 
+        if(self.sw_fixft):
+            w_th_ft  = self.gammatheta * self.ws
+            w_q_ft   = self.gammaq     * self.ws
+            w_CO2_ft = self.gammaCO2   * self.ws 
+        else:
+            w_th_ft  = 0.
+            w_q_ft   = 0.
+            w_CO2_ft = 0. 
+        q_pre        = q0      + self.dtcur * self.qtend
+        if q_pre < 0.001:
+            self.qtend = (0.001-q0)/self.dtcur
+            self.dqtend      = self.gammaq     * (self.we*self.l_entrainment + self.wf - self.M) - self.qtend     + w_q_ft
+
+        self.q        = q0      + self.dtcur * self.qtend
+        self.dq       = dq0     + self.dtcur * self.dqtend
+        self.CO2      = CO20    + self.dtcur * self.CO2tend
+        self.dCO2     = dCO20   + self.dtcur * self.dCO2tend
+        self.dz_h     = dz0     + self.dtcur * self.dztend
+            
         # Limit dz to minimal value
         dz0 = 50
         if(self.dz_h < dz0):
             self.dz_h = dz0 
   
         if(self.sw_wind):
-            self.u        = u0      + self.dt * self.utend
-            self.du       = du0     + self.dt * self.dutend
-            self.v        = v0      + self.dt * self.vtend
-            self.dv       = dv0     + self.dt * self.dvtend
- 
+            self.u        = u0      + self.dtcur * self.utend
+            self.du       = du0     + self.dtcur * self.dutend
+            self.v        = v0      + self.dtcur * self.vtend
+            self.dv       = dv0     + self.dtcur * self.dvtend
+
+        if (self.sw_ac is not None) and ('adv' in self.sw_ac):
+
+            for var in ['theta','q','u','v']:
+                #if ((self.z_pro is not None) and (self.__dict__['adv'+var+'_pro'] is not None)):
+
+            # take into account advection for the whole profile
+                
+                self.air_ap[var] = self.air_ap[var] + self.dtcur * self.air_ap['adv'+var]
+
+            var = 'z'
+            #print(self.air_ap[var])
+                #     print(self.air_ap['adv'+var])
+
+
+
+
+            #moving the profile vertically according to the vertical wind
+                #if ((self.air_ap.z is not None) and (self.air_ap.w is not None)):
+
+
+            # air_apvarold = pd.Series(np.array(self.air_ap.z))
+            # print(self.h,self.ws,self.htend,self.dtcur,air_apvarold )
+            # stop
+
+
+                # # recalculate subsidence at the mixed-layer top from the profile. Yet, this would be overwritten from the external forcing.
+                # self.ws = np.interp(self.h , self.z_pro,self.w_pro)
+
+            #As t is updated, we also need to recalculate theta (and R)
+            self.air_ap['R'] = (self.Rd*(1.-self.air_ap.q) + \
+                                                 self.Rv*self.air_ap.q)
+
+            # air_aptheta_old = pd.Series(self.air_ap['theta'])
+            self.air_ap['t'] = \
+                        self.air_ap.theta * \
+                        (self.air_ap.p/self.Ps)**(self.air_ap['R']/self.cp)
+        if (self.sw_ac is not None) and ('w' in self.sw_ac):
+            zidx_first = np.where(self.air_ap.z > self.h)[0][0]
+            self.air_ap.z[zidx_first:] = self.air_ap.z[zidx_first:] + \
+                                         self.dtcur * self.air_ap.w[zidx_first:]
+
+#            print(self.t, self.dtcur,self.dt,self.air_ap.w[zidx_first])
+#            print(self.t, self.dtcur,self.dt,self.htend)
+
+            # # the pressure levels of the profiles are recalculated according to
+            # # there new height (after subsidence)
+            # self.air_ap.p[zidx_first:] = self.air_ap.p[zidx_first:] - \
+            #         self.air_ap.p[zidx_first:]/self.air_ap['R'][zidx_first:]/self.air_ap['t'][zidx_first:] \
+            #         * self.dtcur *  self.air_ap.w[zidx_first:]
+
+            self.air_ap.p[zidx_first:] = self.air_ap.p[zidx_first:] + \
+                    self.dtcur * self.air_ap.wp[zidx_first:]
+
+            #print(pd.DataFrame([self.air_ap.z,air_apvarold]))
+        # note that theta and q itself are updatet by class itself
+
+    
+        if self.sw_ap:
+            # Just for model consistency preservation purposes, we set the
+            # theta variables of the mixed-layer to nan values, since the
+            # mixed-layer values should overwritte by the mixed-layer
+            # calculations of class.
+            self.air_ap['theta'][0:3] = np.nan 
+            self.air_ap['p'][0:3] = np.nan 
+            self.air_ap['q'][0:3] = np.nan 
+            self.air_ap['u'][0:3] = np.nan 
+            self.air_ap['v'][0:3] = np.nan 
+            self.air_ap['t'][0:3] = np.nan 
+            self.air_ap['z'][0:3] = np.nan 
+
+            # Update the vertical profiles: 
+            #   - new mixed layer properties( h, theta, q ...)
+            #   - any data points below the new ixed-layer height are removed
+
+            # Three data points at the bottom that describe the mixed-layer
+            # properties
+            air_ap_head = self.air_ap.iloc[0:3] # make an empty table with similar
+                                           # columns as air_ap
+            # air_ap_head['z'].iloc[0] = 2.
+            # air_ap_head['z'].iloc[1] = self.__dict__['h']
+            # air_ap_head['z'].iloc[2] = self.__dict__['h']
+            air_ap_head.values[:,list(air_ap_head.columns).index('z')] = \
+                        [2.,self.__dict__['h'],self.__dict__['h']]
+            for var in ['theta','q','u','v']:
+
+                air_ap_head.values[:,list(air_ap_head.columns).index(var)] = \
+                        [self.__dict__[var], \
+                         self.__dict__[var], \
+                         self.__dict__[var] + self.__dict__['d'+var]]
+                
+            #print(self.air_ap)
+
+            # This is the remaining profile considering the remaining
+            # datapoints above the mixed layer height
+            air_ap_tail = self.air_ap.iloc[3:]
+            air_ap_tail = air_ap_tail[air_ap_tail.z > self.h]
+
+            # print('h',self.h)
+            # # only select samples monotonically increasing with height
+            # air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+            # air_ap_tail = pd.DataFrame()
+            # theta_low = self.theta
+            # z_low =     self.h
+            # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+            # for ibottom in range(1,len(air_ap_tail_orig)):
+            #     if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z +2.:
+            #         air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],ignore_index=True)
+
+
+
+
+            # make theta increase strong enough to avoid numerical
+            # instability
+            air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+            air_ap_tail = pd.DataFrame()
+            #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+            #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+            theta_low = self.theta
+            z_low =     self.h
+            ibottom = 0
+            itop = 0
+            # print(air_ap_tail_orig)
+            # stop
+
+            # HW: this is the lower limit that we use for gammatheta, which is
+            # there to avoid model crashes. Besides on this limit, the upper
+            # air profile is modified in a way that is still conserves total
+            # quantities of moisture and temperature. The limit is set by trial
+            # and error. The numerics behind the crash should be investigated
+            # so that a cleaner solution can be provided.
+            # self.gammatheta_lower_limit = 0.002
+            while ((itop in range(0,1)) or (itop != ibottom)):
+                theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+                z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+                if (
+                    #(z_mean > (z_low+0.2)) and \
+                    #(theta_mean > (theta_low+0.02) ) and \
+                    (((theta_mean - theta_low)/(z_mean - z_low)) > self.gammatheta_lower_limit)) or \
+                  (itop >= (len(air_ap_tail_orig)-1)) \
+                   :
+
+                    air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+                    ibottom = itop+1
+                    theta_low = air_ap_tail.theta.iloc[-1]
+                    z_low =     air_ap_tail.z.iloc[-1]
+    
+
+                itop +=1
+                # elif  (itop > len(air_ap_tail_orig)-10):
+                #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+                #print(itop,ibottom)
+
+            if itop > 1:
+                    warnings.warn(str(self.t)+"/"+str(self.tsteps)+\
+                          " Warning! Temperature profile was too steep.  Modifying profile: "+ str(itop - 1)+ " measurements were dropped and replaced with their average.") 
+
+
+            self.air_ap = pd.concat((air_ap_head,\
+                                     air_ap_tail,\
+                                     air_ap_tail_orig[itop:])).reset_index().drop('index',\
+                                                                      axis=1)
+
+            if  self.sw_ac:
+                qvalues = \
+                        self.air_ap.values[:,list(self.air_ap.columns).index('q')]
+
+                self.air_ap.values[:,list(self.air_ap.columns).index('R')] = \
+                        (self.Rd*(1.-qvalues) + self.Rv*qvalues)
+                #self.Ph = self.Ps - self.h * self.g * Density(self.T2m,self.Ps,self.q)
+                self.P_h    = self.Ps - self.rho * self.g * self.h
+                self.air_ap.values[:3,list(self.air_ap.columns).index('p')] = \
+                        [self.Ps,  self.P_h, self.P_h-0.1]
+
+                self.air_ap.t = \
+                            self.air_ap.theta * \
+                            (self.air_ap.p/self.Ps)**(self.air_ap['R']/self.cp)
+
+
+        # WARNING: self.sw_ac always requires self.sw_ap for now!!!
+
+
+
+
+        # else:
+            # in the other case, it is updated at the time the statistics are
+            # calculated 
+
+        if (self.sw_ac is not None) and ('adv' in self.sw_ac):
+
+
+            self.P_h    = self.Ps - self.rho * self.g * self.h
+            in_ml = (self.air_ac.p >= self.P_h)
+
+            if in_ml.sum() == 0:
+                warnings.warn(" no circulation points in the mixed layer found. We just take the bottom one.")
+                in_ml = self.air_ac.index == (len(self.air_ac) - 1)
+            for var in ['theta','q','u','v']:
+
+                # calculation of the advection variables for the mixed-layer
+                # these will be used for the next timestep
+                # Warning: w is excluded for now.
+
+                if var == 'theta':
+                    self.__dict__['adv'+var] = \
+                            ((self.air_ac['advt_x'][in_ml] \
+                             + \
+                             self.air_ac['advt_y'][in_ml])* \
+                            self.air_ac['delpdgrav'][in_ml]).sum()/ \
+                            self.air_ac['delpdgrav'][in_ml].sum()
+                else:
+                    self.__dict__['adv'+var] = \
+                            ((self.air_ac['adv'+var+'_x'][in_ml] \
+                             + \
+                             self.air_ac['adv'+var+'_y'][in_ml])* \
+                            self.air_ac['delpdgrav'][in_ml]).sum()/ \
+                            self.air_ac['delpdgrav'][in_ml].sum()
+
+                # calculation of the advection variables for the profile above
+                # the mixed layer (also for the next timestep)
+
+                if var == 'theta':
+                    self.air_ap['adv'+var] = \
+                                        np.interp(self.air_ap.p,\
+                                                  self.air_ac.p,\
+                                                  self.air_ac['advt_x']) \
+                                        + \
+                                        np.interp(self.air_ap.p,\
+                                                  self.air_ac.p, \
+                                                  self.air_ac['advt_y'])
+                else:
+                    self.air_ap['adv'+var] = \
+                                        np.interp(self.air_ap.p,\
+                                                  self.air_ac.p,\
+                                                  self.air_ac['adv'+var+'_x']) \
+                                        + \
+                                        np.interp(self.air_ap.p,\
+                                                  self.air_ac.p, \
+                                                  self.air_ac['adv'+var+'_y'])
+                # if var == 't':
+                #     print(self.air_ap['adv'+var])
+                #     stop
+
+            # as an approximation, we consider that advection of theta in the
+            # mixed layer is equal to advection of t. This is a sufficient
+            # approximation since theta and t are very similar at the surface
+            # pressure.
+
+            # self.__dict__['advtheta'] = self.__dict__['advt']
+
+        if (self.sw_ac is not None) and ('w' in self.sw_ac):
+            # update the vertical wind profile
+            self.air_ap['wp'] = np.interp(self.air_ap.p, \
+                                          self.air_ac.p, \
+                                          self.air_ac['wp'])
+            self.air_ap['R'] = (self.Rd*(1.-self.air_ap.q) + \
+                                                 self.Rv*self.air_ap.q)
+            self.air_ap['rho'] = self.air_ap.p /self.air_ap.R/  self.air_ap.t
+            
+            air_apwold = self.air_ap['w']
+            self.air_ap['w'] = -self.air_ap['wp'] /self.air_ap['rho']/self.g
+            #print('hello w upd')
+
+            # # # WARNING, THIS DOESN't GIVE THE EXPECTED VALUE!!!
+            # # interpolate subsidence x density
+            # self.air_ap['wrho'] = \
+            #            np.interp(self.air_ap.p,\
+            #                      self.air_ach.p,\
+            #                      self.air_ach['wrho']) \
+            #     
+            # self.air_ap['w'] = \
+            #     self.air_ap['wrho']/(self.air_ap.p/ \
+            #                          (self.Rd*(1.-self.air_ap.q) + \
+            #                           self.Rv*self.air_ap.q)* \
+            #                          self.air_ap.TEMP)
+            # # self.wrho = np.interp(self.P_h,\
+            # #                      self.air_ach.p,\
+            # #                      self.air_ach['wrho']) \
+
+
+
+            # Also update the vertical wind at the mixed-layer height
+            # (subsidence)
+            self.ws   = self.air_ap.w[1]
+        #    print('ws',self.ws,self.air_ap.wp[1],self.air_ap.R[1],self.air_ap.t[1],self.air_ap.q[1])
+
+            ## Finally, we update he 
+            #self.__dict__['divU'] = ((self.air_ac['divU_x'][in_ml] \
+            #                        + \
+            #                        self.air_ac['divU_y'][in_ml])* \
+            #            self.air_ac['delpdgrav'][in_ml]).sum()/ \
+            #            self.air_ac['delpdgrav'][in_ml].sum() 
+            
+
+        if self.sw_ap:
+            for var in ['theta','q','u','v']:
+
+                # update of the slope (gamma) for the different variables, for
+                # the next timestep!
+
+                # there is an warning message that tells about dividing through
+                # zero, which we ignore
+
+                with np.errstate(divide='ignore'):
+                    gammavar = list(np.array(self.air_ap[var][1:].values - \
+                                             self.air_ap[var][:-1].values) \
+                                    / np.array(self.air_ap['z'][1:].values - \
+                                               self.air_ap['z'][:-1].values))
+
+                    # add last element twice (since we have one element less)
+                gammavar.append(gammavar[-1])
+                gammavar = np.array(gammavar)
+                self.air_ap['gamma'+var] = gammavar
+
+                # Based on the above, update the gamma value at the mixed-layer
+                # top
+                # gammatheta, gammaq, gammau, gammav are updated here.
+                self.__dict__['gamma'+var] = self.air_ap['gamma'+var][np.where(self.h >=
+                                                                     self.air_ap.z)[0][-1]]
+
+            
     def run_radiation(self):
         sda    = 0.409 * np.cos(2. * np.pi * (self.doy - 173.) / 365.)
         sinlea = np.sin(2. * np.pi * self.lat / 360.) * np.sin(sda) - np.cos(2. * np.pi * self.lat / 360.) * np.cos(sda) * np.cos(2. * np.pi * (self.t * self.dt + self.tstart * 3600.) / 86400. - 2. * np.pi * self.lon / 360.)
@@ -531,14 +1501,35 @@ def run_radiation(self):
         Tr  = (0.6 + 0.2 * sinlea) * (1. - 0.4 * self.cc)
   
         self.Swin  = self.S0 * Tr * sinlea
+        self.Swin_cs = self.S0 * (0.6 + 0.2 * sinlea)  * sinlea 
+
+        # Swin/Swin_cs = (1. - 0.4*self.cc)
+        # self.cc = (1.-Swin/Swin_cs)/0.4
+
         self.Swout = self.alpha * self.S0 * Tr * sinlea
+        
+        
         self.Lwin  = 0.8 * self.bolz * Ta ** 4.
         self.Lwout = self.bolz * self.Ts ** 4.
           
         self.Q     = self.Swin - self.Swout + self.Lwin - self.Lwout
+        #print('Q',self.Q,self.Swin,self.Swout,self.Lwin,self.Lwout)
   
     def run_surface_layer(self):
-        ueff           = max(0.01, np.sqrt(self.u**2. + self.v**2. + self.wstar**2.))
+        # HW: I had to raise the minimum wind speed to make the simulation with
+        # the non-iterative solution stable (this solution was a wild guess, so I don't
+        # know the exact problem of the instability in case of very low wind
+        # speeds yet)
+        #ueff           = max(0.01, np.sqrt(self.u**2. + self.v**2. + self.wstar**2.))
+
+        # version of 20180730 where there are still some runs crashing. Maybe
+        # an upper limit should be set on the monin-obukhov length instead of
+        # a lower limmit on the wind speed?
+        #ueff           = max(0.1, np.sqrt(self.u**2. + self.v**2. + self.wstar**2.))
+
+        ueff           = max(0.5, np.sqrt(self.u**2. + self.v**2. + self.wstar**2.))
+
+        
         self.thetasurf = self.theta + self.wtheta / (self.Cs * ueff)
         qsatsurf       = qsat(self.thetasurf, self.Ps)
         cq             = (1. + self.Cs * ueff * self.rs) ** -1.
@@ -548,23 +1539,56 @@ def run_surface_layer(self):
   
         zsl       = 0.1 * self.h
         self.Rib  = self.g / self.thetav * zsl * (self.thetav - self.thetavsurf) / ueff**2.
-        self.Rib  = min(self.Rib, 0.2)
+        
 
-        self.L     = self.ribtol(self.Rib, zsl, self.z0m, self.z0h)  # Slow python iteration
-        #self.L    = ribtol.ribtol(self.Rib, zsl, self.z0m, self.z0h) # Fast C++ iteration
- 
-        self.Cm   = self.k**2. / (np.log(zsl / self.z0m) - self.psim(zsl / self.L) + self.psim(self.z0m / self.L)) ** 2.
-        self.Cs   = self.k**2. / (np.log(zsl / self.z0m) - self.psim(zsl / self.L) + self.psim(self.z0m / self.L)) / (np.log(zsl / self.z0h) - self.psih(zsl / self.L) + self.psih(self.z0h / self.L))
-  
-        self.ustar = np.sqrt(self.Cm) * ueff
-        self.uw    = - self.Cm * ueff * self.u
-        self.vw    = - self.Cm * ueff * self.v
- 
-        # diagnostic meteorological variables
-        self.T2m    = self.thetasurf - self.wtheta / self.ustar / self.k * (np.log(2. / self.z0h) - self.psih(2. / self.L) + self.psih(self.z0h / self.L))
-        self.q2m    = self.qsurf     - self.wq     / self.ustar / self.k * (np.log(2. / self.z0h) - self.psih(2. / self.L) + self.psih(self.z0h / self.L))
-        self.u2m    =                - self.uw     / self.ustar / self.k * (np.log(2. / self.z0m) - self.psim(2. / self.L) + self.psim(self.z0m / self.L))
-        self.v2m    =                - self.vw     / self.ustar / self.k * (np.log(2. / self.z0m) - self.psim(2. / self.L) + self.psim(self.z0m / self.L))
+
+        if self.sw_lit:
+            self.Rib  = min(self.Rib, 0.2)
+            self.L     = ribtol(self.Rib, zsl, self.z0m, self.z0h)  # Slow python iteration
+            self.zeta  = zsl/self.L
+            #self.L    = ribtol.ribtol(self.Rib, zsl, self.z0m, self.z0h) # Fast C++ iteration
+            
+        
+            self.Cm   = self.k**2. / (np.log(zsl / self.z0m) - psim(self.zeta) + psim(self.z0m / zsl* self.zeta)) ** 2.
+            self.Cs   = self.k**2. / (np.log(zsl / self.z0m) - psim(self.zeta) + psim(self.z0m / zsl* self.zeta)) / (np.log(zsl / self.z0h) - self.psih(self.zeta) + self.psih(self.z0h / zsl* self.zeta))
+            
+            
+            self.ustar = np.sqrt(self.Cm) * ueff
+            self.uw    = - self.Cm * ueff * self.u
+            self.vw    = - self.Cm * ueff * self.v
+        
+     
+            # diagnostic meteorological variables
+            self.T2m    = self.thetasurf - self.wtheta / self.ustar / self.k * (np.log(2. / self.z0h) - psih(2. / zsl* self.zeta) + psih(self.z0h / zsl* self.zeta))
+            self.q2m    = self.qsurf     - self.wq     / self.ustar / self.k * (np.log(2. / self.z0h) - psih(2. / zsl* self.zeta) + psih(self.z0h / zsl* self.zeta))
+            self.u2m    =                - self.uw     / self.ustar / self.k * (np.log(2. / self.z0m) - psim(2. / zsl* self.zeta) + psim(self.z0m / zsl* self.zeta))
+            self.v2m    =                - self.vw     / self.ustar / self.k * (np.log(2. / self.z0m) - psim(2. / zsl* self.zeta) + self.psim(self.z0m / zsl* self.zeta))
+            
+            # diagnostic meteorological variables
+        else:
+            
+            ## circumventing any iteration with Wouters et al., 2012
+            self.zslz0m = np.max((zsl/self.z0m,10.))
+            #self.Rib  = self.Rib / zsl*self.z0m *self.zslz0m
+            self.zeta = zeta_hs2(self.Rib, self.zslz0m, np.log(self.z0m/self.z0h))
+            #print(str(self.t)+'/'+str(self.tsteps)+' zeta: ',self.zeta,self.Rib, zsl,self.z0m,self.z0h)
+            self.L = zsl/self.zeta
+            funm,funh = funcsche(self.zeta,self.zslz0m, np.log(self.z0m/self.z0h))
+        
+            self.Cm = self.k**2.0/funm/funm
+            self.Cs = self.k**2.0/funm/funh
+            
+            self.ustar = np.sqrt(self.Cm) * ueff
+            self.uw    = - self.Cm * ueff * self.u
+            self.vw    = - self.Cm * ueff * self.v
+            
+            # extrapolation from mixed layer (instead of from surface) to 2meter
+            self.T2m    = self.theta - self.wtheta / self.ustar / self.k * funh
+            self.q2m    = self.q     - self.wq     / self.ustar / self.k * funh
+            self.u2m    =                - self.uw     / self.ustar / self.k * funm
+            self.v2m    =                - self.vw     / self.ustar / self.k * funm
+        
+        
         self.esat2m = 0.611e3 * np.exp(17.2694 * (self.T2m - 273.16) / (self.T2m - 35.86))
         self.e2m    = self.q2m * self.Ps / 0.622
      
@@ -575,6 +1599,7 @@ def ribtol(self, Rib, zsl, z0m, z0h):
         else:
             L  = -1.
             L0 = -2.
+        #print(Rib,zsl,z0m,z0h)
         
         while (abs(L - L0) > 0.001):
             L0      = L
@@ -586,8 +1611,8 @@ def ribtol(self, Rib, zsl, z0m, z0h):
                       - (-zsl /  Lend   * (np.log(zsl / z0h) - self.psih(zsl / Lend  ) + self.psih(z0h / Lend  )) / \
                                           (np.log(zsl / z0m) - self.psim(zsl / Lend  ) + self.psim(z0m / Lend  ))**2.) ) / (Lstart - Lend)
             L       = L - fx / fxdif
-
-            if(abs(L) > 1e15):
+            #print(L)
+            if(abs(L) > 1e12):
                 break
 
         return L
@@ -630,7 +1655,11 @@ def jarvis_stewart(self):
         f3 = 1. / np.exp(- self.gD * (self.esat - self.e) / 100.)
         f4 = 1./ (1. - 0.0016 * (298.0-self.theta)**2.)
   
+        #if np.isnan(self.LAI):
+
         self.rs = self.rsmin / self.LAI * f1 * f2 * f3 * f4
+        # print(self.rs,self.LAI,f1,f2,f3,f4)
+        # stop
 
     def factorial(self,k):
         factorial = 1
@@ -731,11 +1760,14 @@ def ags(self):
     def run_land_surface(self):
         # compute ra
         ueff = np.sqrt(self.u ** 2. + self.v ** 2. + self.wstar**2.)
+        #print('ueff',self.u,self.v,self.wstar)
 
         if(self.sw_sl):
           self.ra = (self.Cs * ueff)**-1.
         else:
           self.ra = ueff / max(1.e-3, self.ustar)**2.
+        # print(self.ra,self.Cs,ueff)
+
 
         # first calculate essential thermodynamic variables
         self.esat    = esat(self.theta)
@@ -759,6 +1791,7 @@ def run_land_surface(self):
         self.rssoil = self.rssoilmin * f2 
  
         Wlmx = self.LAI * self.Wmax
+        #print('Wlmx',Wlmx,self.LAI,self.Wmax,self.Wl)
         self.cliq = min(1., self.Wl / Wlmx) 
      
         # calculate skin temperature implictly
@@ -769,6 +1802,11 @@ def run_land_surface(self):
             / (self.rho * self.cp / self.ra + self.cveg * (1. - self.cliq) * self.rho * self.Lv / (self.ra + self.rs) * self.dqsatdT \
             + (1. - self.cveg) * self.rho * self.Lv / (self.ra + self.rssoil) * self.dqsatdT + self.cveg * self.cliq * self.rho * self.Lv / self.ra * self.dqsatdT + self.Lambda)
 
+        # print('Ts',self.Ts,self.Q,self.rho,self.cp,self.ra,self.theta)
+        # print('Ts',self.cveg, self.cliq,self.Lv,self.Lambda,self.dqsatdT)
+        # print('Ts',self.rs)
+        #print(self.air_ap.p)
+
         esatsurf      = esat(self.Ts)
         self.qsatsurf = qsat(self.Ts, self.Ps)
 
@@ -780,6 +1818,11 @@ def run_land_surface(self):
   
         self.LE     = self.LEsoil + self.LEveg + self.LEliq
         self.H      = self.rho * self.cp / self.ra * (self.Ts - self.theta)
+
+        # print('ra',self.ra,self.ustar,ueff)
+        # print(self.Cs)
+        # print('H',self.ra,self.Ts,self.theta)
+
         self.G      = self.Lambda * (self.Ts - self.Tsoil)
         self.LEpot  = (self.dqsatdT * (self.Q - self.G) + self.rho * self.cp / self.ra * (self.qsat - self.q)) / (self.dqsatdT + self.cp / self.Lv)
         self.LEref  = (self.dqsatdT * (self.Q - self.G) + self.rho * self.cp / self.ra * (self.qsat - self.q)) / (self.dqsatdT + self.cp / self.Lv * (1. + self.rsmin / self.LAI / self.ra))
@@ -796,6 +1839,7 @@ def run_land_surface(self):
   
         # calculate kinematic heat fluxes
         self.wtheta   = self.H  / (self.rho * self.cp)
+        #print('wtheta',self.wtheta,self.H,self.rho,self.cp)
         self.wq       = self.LE / (self.rho * self.Lv)
  
     def integrate_land_surface(self):
@@ -804,16 +1848,37 @@ def integrate_land_surface(self):
         wg0           = self.wg
         Wl0           = self.Wl
   
-        self.Tsoil    = Tsoil0  + self.dt * self.Tsoiltend
-        self.wg       = wg0     + self.dt * self.wgtend
-        self.Wl       = Wl0     + self.dt * self.Wltend
+        self.Tsoil    = Tsoil0  + self.dtcur * self.Tsoiltend
+        self.wg       = wg0     + self.dtcur * self.wgtend
+        self.Wl       = Wl0     + self.dtcur * self.Wltend
   
     # store model output
     def store(self):
         t                      = self.t
-        self.out.t[t]          = t * self.dt / 3600. + self.tstart
+        
+        self.out.time[t]          = t * self.dt / 3600. + self.tstart
+
+        # in case we are at the end of the simulation, we store the vertical
+        # profiles to the output
+        
+        # if t == (len(self.out.time) - 1):
+        #     self.out.air_ac = self.air_ac
+        #     self.out.air_ap = self.air_ap
+
+        
+        # this way, we only need to define the output variables in the output class, so we don't need to specify het again here.
+        #  for key in self.out.__dict__.keys():
+        #      if key in self.__dict__:
+        #          self.out.__dict__[key][t]  = self.__dict__[key]
+        
         self.out.h[t]          = self.h
         
+        # HW20171003 note: most of these updates could also be done with the self.out.__dict__ and self.__dict__ , namely with the key-loop above:
+        
+        self.out.gammatheta[t] = self.gammatheta
+        self.out.gammau[t]     = self.gammau
+        self.out.gammav[t]     = self.gammav
+        self.out.gammaq[t]     = self.gammaq
         self.out.theta[t]      = self.theta
         self.out.thetav[t]     = self.thetav
         self.out.dtheta[t]     = self.dtheta
@@ -822,6 +1887,11 @@ def store(self):
         self.out.wthetav[t]    = self.wthetav
         self.out.wthetae[t]    = self.wthetae
         self.out.wthetave[t]   = self.wthetave
+
+        self.out.advtheta[t]   = self.advtheta
+        self.out.advu[t]       = self.advu
+        self.out.advv[t]       = self.advv
+        self.out.advq[t]       = self.advq
         
         self.out.q[t]          = self.q
         self.out.dq[t]         = self.dq
@@ -855,6 +1925,12 @@ def store(self):
         self.out.v2m[t]        = self.v2m
         self.out.e2m[t]        = self.e2m
         self.out.esat2m[t]     = self.esat2m
+
+
+        self.out.Tsoil[t]      = self.Tsoil
+        self.out.T2[t]         = self.T2
+        self.out.Ts[t]         = self.Ts
+        self.out.wg[t]         = self.wg
         
         self.out.thetasurf[t]  = self.thetasurf
         self.out.thetavsurf[t] = self.thetavsurf
@@ -866,6 +1942,7 @@ def store(self):
         self.out.Rib[t]        = self.Rib
   
         self.out.Swin[t]       = self.Swin
+        self.out.Swin_cs[t]       = self.Swin_cs
         self.out.Swout[t]      = self.Swout
         self.out.Lwin[t]       = self.Lwin
         self.out.Lwout[t]      = self.Lwout
@@ -882,12 +1959,19 @@ def store(self):
         self.out.LEref[t]      = self.LEref
         self.out.G[t]          = self.G
 
+        self.out.ws[t]         = self.ws
+        self.out.advtheta[t]   = self.advtheta
+        self.out.advu[t]       = self.advu
+        self.out.advv[t]       = self.advv
+        self.out.advq[t]       = self.advq
+
         self.out.zlcl[t]       = self.lcl
         self.out.RH_h[t]       = self.RH_h
 
         self.out.ac[t]         = self.ac
         self.out.M[t]          = self.M
         self.out.dz[t]         = self.dz_h
+        self.out.substeps[t]   = self.substeps
   
     # delete class variables to facilitate analysis in ipython
     def exitmodel(self):
@@ -985,6 +2069,7 @@ def exitmodel(self):
         del(self.tstart)
    
         del(self.Swin)
+        del(self.Swin_cs)
         del(self.Swout)
         del(self.Lwin)
         del(self.Lwout)
@@ -1042,12 +2127,16 @@ def exitmodel(self):
 # class for storing mixed-layer model output data
 class model_output:
     def __init__(self, tsteps):
-        self.t          = np.zeros(tsteps)    # time [s]
+        self.time          = np.zeros(tsteps)    # time [s]
 
         # mixed-layer variables
         self.h          = np.zeros(tsteps)    # ABL height [m]
         
         self.theta      = np.zeros(tsteps)    # initial mixed-layer potential temperature [K]
+        self.gammatheta = np.zeros(tsteps)    # initial mixed-layer potential temperature [K]
+        self.gammaq     = np.zeros(tsteps)    # initial mixed-layer potential temperature [K]
+        self.gammau     = np.zeros(tsteps)
+        self.gammav     = np.zeros(tsteps)
         self.thetav     = np.zeros(tsteps)    # initial mixed-layer virtual potential temperature [K]
         self.dtheta     = np.zeros(tsteps)    # initial potential temperature jump at h [K]
         self.dthetav    = np.zeros(tsteps)    # initial virtual potential temperature jump at h [K]
@@ -1055,6 +2144,11 @@ def __init__(self, tsteps):
         self.wthetav    = np.zeros(tsteps)    # surface kinematic virtual heat flux [K m s-1]
         self.wthetae    = np.zeros(tsteps)    # entrainment kinematic heat flux [K m s-1]
         self.wthetave   = np.zeros(tsteps)    # entrainment kinematic virtual heat flux [K m s-1]
+
+        self.advtheta   = np.zeros(tsteps)
+        self.advu       = np.zeros(tsteps)
+        self.advv       = np.zeros(tsteps)
+        self.advq       = np.zeros(tsteps)
         
         self.q          = np.zeros(tsteps)    # mixed-layer specific humidity [kg kg-1]
         self.dq         = np.zeros(tsteps)    # initial specific humidity jump at h [kg kg-1]
@@ -1090,6 +2184,12 @@ def __init__(self, tsteps):
         self.e2m        = np.zeros(tsteps)    # 2m vapor pressure [Pa]
         self.esat2m     = np.zeros(tsteps)    # 2m saturated vapor pressure [Pa]
 
+        # ground variables
+        self.Tsoil       = np.zeros(tsteps)
+        self.T2          = np.zeros(tsteps)
+        self.Ts          = np.zeros(tsteps)
+        self.wg          = np.zeros(tsteps)
+
         # surface-layer variables
         self.thetasurf  = np.zeros(tsteps)    # surface potential temperature [K]
         self.thetavsurf = np.zeros(tsteps)    # surface virtual potential temperature [K]
@@ -1104,6 +2204,7 @@ def __init__(self, tsteps):
 
         # radiation variables
         self.Swin       = np.zeros(tsteps)    # incoming short wave radiation [W m-2]
+        self.Swin_cs    = np.zeros(tsteps)    # incoming short wave radiation clearsky [W m-2]
         self.Swout      = np.zeros(tsteps)    # outgoing short wave radiation [W m-2]
         self.Lwin       = np.zeros(tsteps)    # incoming long wave radiation [W m-2]
         self.Lwout      = np.zeros(tsteps)    # outgoing long wave radiation [W m-2]
@@ -1121,6 +2222,13 @@ def __init__(self, tsteps):
         self.LEref      = np.zeros(tsteps)    # reference evaporation at rs = rsmin / LAI [W m-2]
         self.G          = np.zeros(tsteps)    # ground heat flux [W m-2]
 
+        self.ws         = np.zeros(tsteps)
+        self.advtheta   = np.zeros(tsteps)
+        self.advu       = np.zeros(tsteps)
+        self.advv       = np.zeros(tsteps)
+        self.advq       = np.zeros(tsteps)
+
+
         # Mixed-layer top variables
         self.zlcl       = np.zeros(tsteps)    # lifting condensation level [m]
         self.RH_h       = np.zeros(tsteps)    # mixed-layer top relative humidity [-]
@@ -1129,10 +2237,20 @@ def __init__(self, tsteps):
         self.ac         = np.zeros(tsteps)    # cloud core fraction [-]
         self.M          = np.zeros(tsteps)    # cloud core mass flux [m s-1]
         self.dz         = np.zeros(tsteps)    # transition layer thickness [m]
+        
+        
+        self.substeps   = np.zeros(tsteps)    # number of additional substep time integrations needed [-]
 
 # class for storing mixed-layer model input data
 class model_input:
     def __init__(self):
+
+        # # comment not valid
+        # we comment out the initialization, because there is a problem when
+        # inheriting values from one the another class4gl_iput. We also expect
+        # that the user specifies all the required parmameters (if not, an error
+        # is raised). 
+
         # general model variables
         self.runtime    = None  # duration of model run [s]
         self.dt         = None  # time step [s]
@@ -1147,6 +2265,10 @@ def __init__(self):
         self.fc         = None  # Coriolis parameter [s-1]
         
         self.theta      = None  # initial mixed-layer potential temperature [K]
+        #self.air_ap.THTA  = None  # optional/initial profile of potential temperature [K]
+
+        #self.z_pro      = None  # height coordinate of the optional input profiles [m]
+
         self.dtheta     = None  # initial temperature jump at h [K]
         self.gammatheta = None  # free atmosphere potential temperature lapse rate [K m-1]
         self.advtheta   = None  # advection of heat [K s-1]
@@ -1154,6 +2276,9 @@ def __init__(self):
         self.wtheta     = None  # surface kinematic heat flux [K m s-1]
         
         self.q          = None  # initial mixed-layer specific humidity [kg kg-1]
+        #self.q_pro      = None  # optional/initial profile of specific humidity [kg kg-1]
+        #self.p_pro      = None  # optional/initial profile of pressure, just for diagnosis purposes [Pa]
+
         self.dq         = None  # initial specific humidity jump at h [kg kg-1]
         self.gammaq     = None  # free atmosphere specific humidity lapse rate [kg kg-1 m-1]
         self.advq       = None  # advection of moisture [kg kg-1 s-1]
@@ -1238,3 +2363,8 @@ def __init__(self):
         # Cumulus parameters
         self.sw_cu      = None  # Cumulus parameterization switch
         self.dz_h       = None  # Transition layer thickness [m]
+        
+# BEGIN -- HW 20171027
+        # self.cala       = None      # soil heat conductivity [W/(K*m)]
+        # self.crhoc      = None      # soil heat capacity  [J/K*m**3]
+# END -- HW 20171027
diff --git a/class4gl/processing/batch_update_output.py b/class4gl/processing/batch_update_output.py
new file mode 100644
index 0000000..ab51951
--- /dev/null
+++ b/class4gl/processing/batch_update_output.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+
+"""
+Usage:
+python batch_update.py --exec $CLASS4GL/simulations/update_yaml_old.py
+--path_experiments $VSC_DATA_VO/D2D/data/C4GL/GLOBAL_NOAC/ --path_input
+$VSC_DATA_VO/D2D/data/C4GL/GLOBAL_NOAC_BACKUP_20180904/ --c4gl_path_lib
+$CLASS4GL --split_by 50 --global_keys "KGC" --subset_input morning --experiments
+"GLOBAL_NOAC"
+"""
+
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+parser = argparse.ArgumentParser()
+#if __name__ == '__main__':
+parser.add_argument('--exec') # chunk simulation script
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--pbs_string',default=' -l walltime=2:0:0')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--error_handling')
+parser.add_argument('--subset_input',default='morning') 
+                                        # this tells which yaml subset
+                                        # to initialize with.
+                                        # Most common options are
+                                        # 'morning' and 'ini'.
+parser.add_argument('--subset_output',default='morning') 
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime')
+# delete folders of experiments before running them
+parser.add_argument('--split_by',default=50)# station soundings are split
+                                            # up in chunks
+
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--path_input') #,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output') #,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+
+
+#arguments only used for update_yaml.py
+parser.add_argument('--path_dataset') 
+parser.add_argument('--global_keys') 
+parser.add_argument('--updates') 
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+print("getting all stations from --path_input")
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_input,suffix=args.subset_input,refetch_stations=False)
+
+print('defining all_stations_select')
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+if args.station_id is not None:
+    print("Selecting stations by --station_id")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table [--first_station_row,--last_station_row]")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         args.path_input,\
+                                         subset=args.subset_input,\
+                                         refetch_records=False,\
+                                        )
+
+print('splitting batch in --split_by='+args.split_by+' jobs.')
+totalchunks = 0
+for istation,current_station in all_stations_select.iterrows():
+    records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+    chunks_current_station = len(records_morning_station_select.query('STNID == '+str(current_station.name)).chunk.unique())
+    totalchunks +=chunks_current_station
+
+print('total chunks (= size of array-job) per experiment: ' + str(totalchunks))
+
+#if sys.argv[1] == 'qsub':
+# with qsub
+
+
+
+#C4GLJOB_timestamp="+dt.datetime.now().isoformat()+",
+command = 'qsub '+args.pbs_string+' '+args.c4gl_path_lib+'/simulations/batch_simulations.pbs -t 0-'+\
+            str(totalchunks-1)+" -v '"
+# propagate arguments towards the job script
+first = True
+for argkey in args.__dict__.keys():
+    if ((argkey not in ['pbs_string']) and \
+        # default values are specified in the simulation script, so
+        # excluded here
+        (args.__dict__[argkey] is not None)
+       ):
+        if first:
+            command +='C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+        else:
+            command +=',C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+    first = False
+
+command = command+"'"
+print('Submitting array job: '+command)
+os.system(command)
+
+
+    #os.system(command)
+# elif sys.argv[1] == 'wsub':
+#     
+#     # with wsub
+#     STNlist = list(df_stations.iterrows())
+#     NUMSTNS = len(STNlist)
+#     PROCS = NUMSTNS 
+#     BATCHSIZE = 1 #math.ceil(np.float(NUMSTNS)/np.float(PROCS))
+# 
+#     os.system('wsub -batch /user/data/gent/gvo000/gvo00090/D2D/scripts/C4GL/global_run.pbs -t 0-'+str(PROCS-1))
+
diff --git a/class4gl/processing/update_output.py b/class4gl/processing/update_output.py
new file mode 100644
index 0000000..8fab56b
--- /dev/null
+++ b/class4gl/processing/update_output.py
@@ -0,0 +1,327 @@
+# -*- coding: utf-8 -*-
+
+""" 
+Purpose:
+    update variables in class4gl yaml files, eg., when you need new categorical
+    values in the table.
+
+
+"""
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+import dateutil.parser
+
+import argparse
+
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+parser.add_argument('--path_input')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--path_output')
+parser.add_argument('--diag_tropo',default=None)#['advt','advq','advu','advv'])
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--mode',default='ini') # run a specific station id
+# this is the type of the yaml that needs to be updated. Can be 'ini' or 'mod'
+parser.add_argument('--updates')
+parser.add_argument('--subset_input',default='morning') # this tells which yaml subset
+parser.add_argument('--subset_output',default='morning') # this tells which yaml subset
+                                                      # to update in the yaml
+                                                      # dataset.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+
+parser.add_argument('--split_by',default=-1)# station soundings are split
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--global_keys') 
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+# iniitialize global data
+globaldata = data_global()
+if 'era_profiles' in args.updates.strip().split(","):
+    globaldata.sources = {**globaldata.sources,**{
+            "ERAINT:t"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/t_6hourly/t_*_6hourly.nc",
+            "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q_*_6hourly.nc",
+            "ERAINT:u"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/u_6hourly/u_*_6hourly.nc",
+            "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v_*_6hourly.nc",
+            }}
+
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+
+print("getting stations")
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_input,suffix=args.subset_input,refetch_stations=False)
+
+print('defining all_stations_select')
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+if args.station_id is not None:
+    print("Selecting station by ID")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         args.path_input,\
+                                         subset=args.subset_input,
+                                         refetch_records=False,
+                                         )
+
+# only run a specific chunck from the selection
+if args.global_chunk_number is not None:
+    if args.station_chunk_number is not None:
+        raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+
+
+    # if not (int(args.split_by) > 0) :
+    #         raise ValueError("global_chunk_number is specified, but --split-by is not a strict positive number, so I don't know how to split the batch into chunks.")
+
+    run_station_chunk = None
+    print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+    totalchunks = 0
+    stations_iter = all_stations_select.iterrows()
+    in_current_chunk = False
+    try:
+        while not in_current_chunk:
+            istation,current_station = stations_iter.__next__()
+            all_records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+            #chunks_current_station = math.ceil(float(len(all_records_morning_station_select))/float(args.split_by))
+
+            chunks_current_station = len(all_records_morning_station_select.query('STNID == '+str(current_station.name)).chunk.unique())
+            print('chunks_current_station',chunks_current_station)
+
+            in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+        
+            if in_current_chunk:
+                run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                run_station_chunk =all_records_morning_station_select.query('STNID == '+str(current_station.name)).chunk.unique()[int(args.global_chunk_number) - totalchunks ]
+        
+            totalchunks +=chunks_current_station
+        
+
+    except StopIteration:
+       raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+    print("station = ",list(run_stations.index))
+    print("station chunk number:",run_station_chunk)
+
+# if no global chunk is specified, then run the whole station selection in one run, or
+# a specific chunk for each selected station according to # args.station_chunk_number
+else:
+    run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+    if args.station_chunk_number is not None:
+        run_station_chunk = int(args.station_chunk_number)
+        print("station(s) that is processed.",list(run_stations.index))
+        print("chunk number: ",run_station_chunk)
+    else:
+        if args.split_by != -1:
+            raise ValueError("Chunks are defined by --split-by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split-by.")
+        run_station_chunk = 0
+        print("stations that are processed.",list(run_stations.index))
+        
+
+#print(all_stations)
+print('Fetching current records')
+records_input = get_records(run_stations,\
+                              args.path_input,\
+                              subset=args.subset_input,
+                              refetch_records=False,
+                              )
+
+# if args.timestamp is None:
+#     backupdir = args.path_input+'/'+dt.datetime.now().isoformat()+'/'
+# else: 
+#     backupdir = args.path_input+'/'+args.timestamp+'/'
+# print('creating backup dir: '+backupdir)
+# os.system('mkdir -p "'+backupdir+'"')
+
+
+os.system('mkdir -p '+args.path_output)
+
+for istation,current_station in run_stations.iterrows():
+    records_input_station = records_input.query('STNID == ' +\
+                                                    str(current_station.name))
+
+    records_input_station_chunk = records_input_station.query('STNID == ' +\
+                                                    str(current_station.name)+\
+                                                   '& chunk == '+str(run_station_chunk))
+    print('lenrecords_input_station_chunk: ',len(records_input_station_chunk))
+    print('split_by*run_station_chunk',int(args.split_by) * int(run_station_chunk))
+    print('split_by*run_station_chunk+1',int(args.split_by) * int(run_station_chunk+1))
+    
+    # if (int(args.split_by) * int(run_station_chunk)) >= (len(records_forcing_station)):
+    #     print("warning: outside of profile number range for station "+\
+    #           str(current_station)+". Skipping chunk number for this station.")
+    if len(records_input_station_chunk) == 0:
+        print("warning: outside of profile number range for station "+\
+              str(current_station)+". Skipping chunk number for this station.")
+    else:
+        # normal case
+        if ((int(args.split_by) > 0) or \
+            (os.path.isfile(args.path_input+'/'+format(current_station.name,'05d')+'_'+\
+                 str(run_station_chunk)+'_'+args.subset_input+'.yaml'))):
+            fn_input = \
+                    args.path_input+'/'+format(current_station.name,'05d')+'_'+\
+                    str(run_station_chunk)+'_'+args.subset_input+'.yaml'
+            file_input = \
+                open(fn_input,'r')
+            fn_output = args.path_output+'/'+'/'+format(current_station.name,'05d')+'_'+\
+                     str(run_station_chunk)+'_'+args.subset_output+'.yaml'
+            file_output = \
+                open(fn_output,'w')
+            # fn_forcing_pkl = args.path_forcing+'/'+format(current_station.name,'05d')+'_'+\
+            #          str(run_station_chunk)+'_'+args.subset_forcing+'.pkl'
+
+            # fn_backup = backupdir+format(current_station.name,'05d')+'_'+\
+            #          str(run_station_chunk)+'_'+args.subset_forcing+'.yaml'
+            # fn_backup_pkl = backupdir+format(current_station.name,'05d')+'_'+\
+            #          str(run_station_chunk)+'_'+args.subset_forcing+'.pkl'
+        else:
+            print("\
+Warning. We are choosing chunk 0 without specifying it in filename.    \
+ No-chunk naming will be removed in the future."\
+                 )
+
+            fn_input = \
+                    args.path_input+'/'+format(current_station.name,'05d')+'_'+\
+                    args.subset_input+'.yaml'
+            file_input = \
+                open(fn_input,'r')
+            fn_output = args.path_output+'/'+'/'+format(current_station.name,'05d')+'_'+\
+                     str(run_station_chunk)+'_'+args.subset_output+'.yaml'
+            file_output = \
+                open(fn_output,'w')
+            # fn_forcing_pkl = args.path_forcing+format(current_station.name,'05d')+'_'+\
+            #          str(run_station_chunk)+'_'+args.subset_forcing+'.pkl'
+
+            # fn_backup = backupdir+format(current_station.name,'05d')+'_'+\
+            #          str(run_station_chunk)+'_'+args.subset_forcing+'.yaml'
+            # fn_backup_pkl = backupdir+format(current_station.name,'05d')+'_'+\
+            #          args.subset_forcing+'.pkl'
+
+        onerun = False
+        print('starting station chunk number: '\
+              +str(run_station_chunk)+'(size: '+str(args.split_by)+' soundings)')
+
+        #records_forcing_station_chunk = records_forcing_station[(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+
+        # records_forcing_station_chunk = records_forcing.query('STNID == ' +\
+        #                                                 str(current_station.name)+\
+        #                                                '& chunk == '+str(run_station_chunk))
+        isim = 0
+        for (STNID,chunk,index),record_input in records_input_station_chunk.iterrows():
+                print('starting '+str(isim+1)+' out of '+\
+                  str(len(records_input_station_chunk) )+\
+                  ' (station total: ',str(len(records_input_station)),')')  
+            
+                c4gli_output = get_record_yaml(file_input, 
+                                                record_input.index_start, 
+                                                record_input.index_end,
+                                                mode=args.mode)
+                if args.diag_tropo is not None:
+                    seltropo = (c4gli_input.air_ac.p > c4gli_input.air_ac.p.iloc[-1]+ 3000.*(- 1.2 * 9.81 ))
+                    profile_tropo = c4gli_input.air_ac[seltropo]
+                    for var in args.diag_tropo:
+                        if var[:3] == 'adv':
+                            mean_adv_tropo = np.mean(profile_tropo[var+'_x']+profile_tropo[var+'_y'] )
+                            c4gli_output.update(source='era-interim',pars={var+'_tropo':mean_adv_tropo})
+                        else:
+                            print("warning: tropospheric variable "+var+" not recognized")
+                if 'era_profiles' in args.updates.strip().split(" "):
+                    c4gli_output.get_global_input(globaldata,only_keys=['t','u','v','q','sp'])
+
+                    c4gli_output.update(source='era-interim',pars={'Ps' : c4gli_output.pars.sp})
+
+                    cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+                    Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+                    Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+                    R = (Rd*(1.-c4gli_output.air_ac.q) + Rv*c4gli_output.air_ac.q)
+                    rho = c4gli_output.air_ac.p/R/c4gli_output.air_ac.t
+                    dz = c4gli_output.air_ac.delpdgrav/rho
+                    z = [dz.iloc[-1]/2.]
+                    for idz in list(reversed(range(0,len(dz)-1,1))):
+                        z.append(z[-1]+ (dz[idz+1]+dz[idz])/2.)
+                    z = list(reversed(z))
+
+                    theta = c4gli_output.air_ac.t * \
+                               (c4gli_output.pars.sp/(c4gli_output.air_ac.p))**(R/cp)
+                    thetav   = theta*(1. + 0.61 * c4gli_output.air_ac.q)
+
+                    
+                    c4gli_output.update(source='era-interim',air_ac=pd.DataFrame({'z':list(z),
+                                                                           'theta':list(theta),
+                                                                           'thetav':list(thetav),
+                                                                          }))
+                    air_ap_input = c4gli_output.air_ac[::-1].reset_index().drop('index',axis=1)
+                    air_ap_mode = 'b'
+                    air_ap_input_source = c4gli_output.query_source('air_ac:theta')
+
+
+                    c4gli_output.mixed_layer_fit(air_ap=air_ap_input,
+                                         source=air_ap_input_source,
+                                         mode=air_ap_mode)
+
+                if not c4gli_output.check_source_globaldata():
+                    print('Warning: some input sources appear invalid')
+
+
+                
+                #print('c4gli_forcing_ldatetime',c4gli_forcing.pars.ldatetime)
+                
+                # if args.global_keys is not None:
+                #     print(args.global_keys.strip(' ').split(' '))
+                #     c4gli_forcing.get_global_input(
+                #         globaldata, 
+                #         only_keys=args.global_keys.strip(' ').split(' ')
+                #     )
+
+                c4gli_output.dump(file_output)
+                    
+                    
+                onerun = True
+                isim += 1
+
+
+        file_input.close()
+        file_output.close()
+
+        if onerun:
+            # os.system('mv "'+fn_forcing+'" "'+fn_backup+'"')
+            # if os.path.isfile(fn_forcing_pkl):
+            #     os.system('mv "'+fn_forcing_pkl+'" "'+fn_backup_pkl+'"')
+            # os.system('mv "'+fn_experiment+'" "'+fn_forcing+'"')
+            # print('mv "'+fn_experiment+'" "'+fn_forcing+'"')
+            records_forcing_current_cache = get_records(pd.DataFrame([current_station]),\
+                                                       args.path_output+'/'+'/',\
+                                                       getchunk = int(run_station_chunk),\
+                                                       subset=args.subset_output,
+                                                       refetch_records=True,
+                                                       )
+
diff --git a/Makefile b/class4gl/ribtol/Makefile
similarity index 100%
rename from Makefile
rename to class4gl/ribtol/Makefile
diff --git a/MakefileMac b/class4gl/ribtol/MakefileMac
similarity index 100%
rename from MakefileMac
rename to class4gl/ribtol/MakefileMac
diff --git a/class4gl/ribtol/__init__.py b/class4gl/ribtol/__init__.py
new file mode 100644
index 0000000..02a6a7d
--- /dev/null
+++ b/class4gl/ribtol/__init__.py
@@ -0,0 +1,7 @@
+from . import ribtol_hw
+
+__version__ = '0.1.0'
+
+__author__ = 'Hendrik Wouters '
+
+__all__ = []
diff --git a/ribtol.cpp b/class4gl/ribtol/ribtol.cpp
similarity index 100%
rename from ribtol.cpp
rename to class4gl/ribtol/ribtol.cpp
diff --git a/ribtol.pyx b/class4gl/ribtol/ribtol.pyx
similarity index 100%
rename from ribtol.pyx
rename to class4gl/ribtol/ribtol.pyx
diff --git a/class4gl/ribtol/ribtol_hw.py b/class4gl/ribtol/ribtol_hw.py
new file mode 100644
index 0000000..1946cc8
--- /dev/null
+++ b/class4gl/ribtol/ribtol_hw.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan 12 10:46:20 2018
+
+@author: vsc42247
+"""
+
+
+
+# purpose of calc_cm_ch: calculate momentum and thermal turbulent diffusion coefficients of the surface layer with a non-iterative procedure (Wouters et al., 2012)
+
+# input:
+
+# zrib = bulk Richardson number = (g/T)* DT * z/(ua^2)
+#   with:
+#     g = 9.81 m/s2 the gravitational acceleration
+#     z = height (in meters) of the surface layer under consideration 
+#     T = (reference) temperature (in Kelvin) at height z 
+#     DT = (T - T_s) = temperature (in Kelvin) gradient between the surface and height z 
+#     u_a^2 = u^2 +  v^2 is the squared horizontal absolute wind speed 
+# zzz0m = ratio z/z0 between the height z and the momentum roughness length z0m
+# zkbm = ln(z0m/z0h), with z0m, z0h the momentum and thermal roughness length, respectively.
+
+# output: diffusion coefficients (CM and CH) which cna be used to determine surface-layer turbulent transport
+# u'w' = - CM ua^2.
+# w'T' = - CH ua DT 
+
+
+# Reference:
+# Wouters, H., De Ridder, K., and Lipzig, N. P. M.: Comprehensive
+# Parametrization of Surface-Layer Transfer Coefficients for Use
+# in Atmospheric Numerical Models, Bound.-Lay. Meteorol., 145,
+# 539–550, doi:10.1007/s10546-012-9744-3, 2012.
+
+import numpy as np
+
+def calc_cm_ch (zeta,zzz0m,zkbm):
+    krm = 0.4
+
+    #ZETA = zeta_hs2(zrib,zzz0m,zkbm)
+    FUNM,FUNH = funcsche(ZETA,zzz0m,zkbm)
+    CM = krm**2.0/FUNM/FUNM
+    CH = krm**2.0/FUNM/FUNH
+
+    # FUNMn,FUNHn = funcsche(0.,zzz0m,zkbm)
+    # CMn = krm**2.0/FUNMn/FUNMn
+    # CHn = krm**2.0/FUNMn/FUNHn
+
+    # print ZETA,FUNM,FUNH
+    # print 'CMCMN',CM/CMn
+    # print 'CHCHN',CH/CHn
+
+    return CM,CH
+
+
+def zeta_hs2(RiB,zzz0m,kBmin1):
+    #print(RiB,zzz0m,kBmin1)
+    mum=2.59
+    muh=0.95
+    nu=0.5
+    lam=1.5
+
+    betah = 5.0
+
+    zzz0h = zzz0m*np.exp(kBmin1)
+    zzzs = zzz0m*0.06 # to be changed!! r. 101 nog bekijken!!
+
+    L0M = np.log(zzz0m)
+    L0H = np.log(zzz0h)
+    facM = np.log(1.+lam/mum/zzzs)*np.exp(-mum*zzzs)/lam
+    facH = np.log(1.+lam/muh/zzzs)*np.exp(-muh*zzzs)/lam
+    L0Ms = L0M + facM 
+    L0Hs = L0H + facH
+
+    if RiB < 0.:
+        p = np.log(1.-RiB)
+        Q = -0.486 +0.219*p - 0.0331*p**2-4.93*np.exp(-L0H) - 3.65/L0H +\
+            0.38*p/L0H+ 14.8/L0H/L0H-0.946*p/L0H/L0H-10.0/L0H**3+ \
+            0.392*L0M/L0H-0.084*p*L0M/L0H+0.368*L0M/L0H/L0H
+        # print 'p: ',p
+        # print 'Q: ',Q
+        zeta = (1. + p*Q)* L0Ms**2/L0Hs * RiB
+    else:
+        betam = 4.76+7.03/zzz0m +0.24*zzz0m/zzz0h # to be changed
+        # betam = 5.0 + 1.59*10.**(-5.)*(np.exp(13.0-L0M)-1.0) \
+        #         +0.24*(np.exp(-kBmin1)-1.0) # to be changed!!
+        # print('betam',betam)
+        lL0M = np.log(L0M)
+        S0Ms = 1.-1./zzz0m + (1.+nu/mum/zzzs)*facM
+        S0Hs = 1.-1./zzz0h + (1.+nu/muh/zzzs)*facH
+        zetat = -0.316-0.515*np.exp(-L0H) + 25.8 *np.exp(-2.*L0H) + 4.36/L0H \
+                -6.39/L0H/L0H+0.834*lL0M - 0.0267*lL0M**2
+        # print('zetat',zetat)
+        RiBt = zetat *(L0Hs+ S0Hs*betah*zetat)/(L0Ms+S0Ms*betam*zetat)**2 
+        # print('RiBt',RiBt)
+
+        if (RiB > RiBt):
+            D = (L0Ms+S0Ms*betam*zetat)**3/\
+                (L0Ms*L0Hs+zetat*(2.*S0Hs * betah * L0Ms - S0Ms*betam*L0Hs))
+            zeta = zetat + D*(RiB-RiBt)
+        else:
+            r = RiB - S0Hs*betah/(S0Ms*betam)**2
+            B = S0Ms*betam*L0Hs- 2.*S0Hs*betah*L0Ms
+            C = 4.*(S0Ms*betam)**2 * L0Ms *(S0Hs*betah*L0Ms-S0Ms*betam*L0Hs)
+            zeta = - L0Ms / S0Ms/betam - B*C/(4.*(S0Ms*betam)**3 *(B**2+abs(C*r)))
+            if r != 0:
+                zeta = zeta + (B-np.sqrt(B**2+C*r) + B*C*r/(2.*(B**2+abs(C*r))))/(2.*(S0Ms*betam)**3*r)
+    # print('zeta',zeta)
+    return zeta
+
+def funcsche(zeta,zzz0,kBmin1):
+
+
+    mum=2.5
+    muh=0.9
+    nu=0.5
+    lam=1.5
+    
+    p2=3.141592/2.
+    
+    lnzzz0=np.log(zzz0)
+    zzzs=zzz0*0.06
+    zetamcorr=(1.+nu/(mum*zzzs))*zeta
+    zetam0=zeta/zzz0
+    zetahcorr=(1.+nu/(muh*zzzs))*zeta
+    zetah0=zeta/(zzz0*np.exp(kBmin1))
+    
+    if (zeta <= 0.):
+    
+        gamma=15.2
+        alfam=0.25
+        xx=(1.-gamma*zeta)**alfam
+        psim=2.*np.log((1.+xx)/2.)+np.log((1.+xx**2.)/2.)-2.*np.arctan(xx)+p2
+        xx0=(1.-gamma*zetam0)**alfam
+        psim0=2.*np.log((1.+xx0)/2.)+np.log((1.+xx0**2.)/2.)-2.*np.arctan(xx0)+p2
+        phimcorr=(1.-gamma*zetamcorr)**(-alfam)
+        
+        alfah=0.5
+        yy=(1.-gamma*zeta)**alfah
+        psih=2.*np.log((1.+yy)/2.)
+        yy0=(1.-gamma*zetah0)**alfah
+        psih0=2.*np.log((1.+yy0)/2.)
+        phihcorr=(1.-gamma*zetahcorr)**(-alfah)
+    else: 
+    
+        aa=6.1
+        bb=2.5
+        psim=-aa*np.log(zeta+(1.+zeta**bb)**(1./bb))
+        psim0=-aa*np.log(zetam0+(1.+zetam0**bb)**(1./bb))
+        phimcorr=1.+aa*(zetamcorr+zetamcorr**bb*(1.+zetamcorr**bb)**((1.-bb)/bb))/(zetamcorr+(1.+zetamcorr**bb)**(1./bb))
+        
+        cc=5.3
+        dd=1.1
+        psih=-cc*np.log(zeta+(1.+zeta**dd)**(1./dd))
+        psih0=-cc*np.log(zetah0+(1.+zetah0**dd)**(1./dd))
+        phihcorr=1.+cc*(zetahcorr+zetahcorr**dd*(1.+zetahcorr**dd)**((1.-dd)/dd))/(zetahcorr+(1.+zetahcorr**dd)**(1./dd))
+    
+    psistrm=phimcorr*(1./lam)*np.log(1.+lam/(mum*zzzs))*np.exp(-mum*zzzs)
+    psistrh=phihcorr*(1./lam)*np.log(1.+lam/(muh*zzzs))*np.exp(-muh*zzzs)
+    
+    funm=lnzzz0-psim+psim0 +psistrm
+    funh=lnzzz0+kBmin1-psih+psih0 +psistrh
+    return funm,funh
+
diff --git a/class4gl/ribtol/setup.py b/class4gl/ribtol/setup.py
new file mode 100644
index 0000000..bfb44db
--- /dev/null
+++ b/class4gl/ribtol/setup.py
@@ -0,0 +1,12 @@
+# build with "python setup.py build_ext --inplace"
+from distutils.core import setup
+from distutils.extension import Extension
+from Cython.Build import cythonize
+import numpy as np
+import os
+
+os.environ["CC"] = "g++-7"
+
+setup(
+    ext_modules = cythonize((Extension("ribtol", sources=["ribtol.pyx"], include_dirs=[np.get_include()], ), ))
+)
diff --git a/class4gl/setup/batch_setup_era.pbs b/class4gl/setup/batch_setup_era.pbs
new file mode 100644
index 0000000..18a6018
--- /dev/null
+++ b/class4gl/setup/batch_setup_era.pbs
@@ -0,0 +1,34 @@
+#!/bin/bash 
+#
+#PBS -j oe
+#PBS -M hendrik.wouters@ugent.be
+#PBS -m b
+#PBS -m e
+#PBS -m a
+#PBS -N c4gl_setup
+
+module purge
+source ~/.bashrc
+
+echo loading modules: $LOADDEPSCLASS4GL 
+$LOADDEPSCLASS4GL 
+
+EXEC_ALL="python $C4GLJOB_exec --global_chunk_number $PBS_ARRAYID"
+
+for var in $(compgen -v | grep C4GLJOB_ ); do
+    echo $var
+    if [ "$var" != "C4GLJOB_exec" ]
+    then
+    EXEC_ALL=$EXEC_ALL" --"`echo $var | cut -c9-`"="${!var}
+    fi
+done
+
+
+# EXEC_ALL="python $exec --global-chunk-number $PBS_ARRAYID \
+#                        --split-by $split_by \
+#                        --dataset $dataset \
+#                        --experiments $experiments"
+#                  #      --path-soundings $path_soundings \
+echo Executing: $EXEC_ALL
+$EXEC_ALL
+
diff --git a/class4gl/setup/batch_setup_era.py b/class4gl/setup/batch_setup_era.py
new file mode 100644
index 0000000..7f8c0d0
--- /dev/null
+++ b/class4gl/setup/batch_setup_era.py
@@ -0,0 +1,188 @@
+
+# -*- coding: utf-8 -*-
+
+import logging
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+import importlib
+spam_loader = importlib.find_loader('Pysolar')
+found = spam_loader is not None
+if found:
+    import Pysolar
+    import Pysolar.util.GetSunriseSunset
+else:
+    import pysolar as Pysolar
+    GetSunriseSunset =  Pysolar.util.get_sunrise_sunset
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--exec') # chunk simulation script
+parser.add_argument('--pbs_string',default='')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--first_YYYYMMDD',default="19810101")
+parser.add_argument('--last_YYYYMMDD',default="20180101")
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--latitude') # run a specific station id
+parser.add_argument('--longitude') # run a specific station id
+parser.add_argument('--error_handling',default='dump_on_success')
+parser.add_argument('--subset_forcing',default='morning') # this tells which yaml subset
+parser.add_argument('--subset_experiments',default='ini') # this tells which yaml subset
+                                                      # to initialize with.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime',default='from_afternoon_profile')
+
+parser.add_argument('--experiments')
+parser.add_argument('--split_by',default=-1)# station soundings are split
+                                            # up in chunks
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+EXP_DEFS  =\
+{
+  'ERA-INTERIM_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'IOPS_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'IOPS_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+
+# iniitialize global data
+# ===============================
+print("Initializing global data")
+# ===============================
+globaldata = data_global()
+globaldata.sources = {**globaldata.sources,**{
+    
+        "ERAINT:t"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/t_6hourly/t_19830609-19830808_6hourly.nc",
+        "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q_19830609-19830808_6hourly.nc",
+        "ERAINT:u"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/u_6hourly/u_19830609-19830808_6hourly.nc",
+        "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v_19830609-19830808_6hourly.nc",
+    
+#        "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q_19830209-19830410_6hourly.nc",
+ #       "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q*_6hourly.nc",
+ #       "ERAINT:u"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/u_6hourly/u*_6hourly.nc",
+ #       "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v*_6hourly.nc",
+        }}
+
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+
+
+# ===============================
+print("getting a list of stations")
+# ===============================
+all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+
+
+# # ===============================
+# print("Selecting station by ID")
+# # ===============================
+# stations_iter = stations_iterator(all_stations)
+# STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+# all_stations_select = pd.DataFrame([run_station])
+# print(run_station)
+
+
+# ====================================
+print('defining all_stations_select')
+# ====================================
+
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+if (args.latitude is not None) or (args.longitude is not None):
+    print('custom coordinates not implemented yet, please ask developer.')
+elif args.station_id is not None:
+    print("Selecting station by ID")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+#     print("making a custom station according to the coordinates")
+# 
+#     STNID = 43.23
+else:
+     print("Selecting stations from a row range in the table")
+     all_stations_select = pd.DataFrame(all_stations.table)
+     if args.last_station_row is not None:
+         all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+     if args.first_station_row is not None:
+         all_stations_select = all_station_select.iloc[int(args.first_station):]
+
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+dtfirst = dt.datetime.strptime(args.first_YYYYMMDD,"%Y%m%d",)
+dtlast = dt.datetime.strptime(args.last_YYYYMMDD,"%Y%m%d",)
+# ===============================
+print("Creating daily timeseries from", dtfirst," to ", dtlast)
+# ===============================
+DTS = [dtfirst + dt.timedelta(days=iday) for iday in \
+       range(int((dtlast + dt.timedelta(days=1) -
+                  dtfirst).total_seconds()/3600./24.))]
+
+if args.split_by != -1:
+    totalchunks = len(all_stations_select)*math.ceil(len(DTS)/int(args.split_by))
+else:
+    totalchunks = len(all_stations_select)
+
+print(totalchunks)
+
+#if args.cleanup_experiments:
+#    os.system("rm -R "+args.path_experiments+'/')
+
+# C4GLJOB_timestamp="+dt.datetime.now().isoformat()+",
+command = 'qsub '+args.pbs_string+' '+args.c4gl_path_lib+'/setup/batch_setup_era.pbs -t 0-'+\
+            str(totalchunks-1)+" -v "
+# propagate arguments towards the job script
+lfirst = True
+for argkey in args.__dict__.keys():
+    if ((argkey not in ['experiments','pbs_string','cleanup_experiments']) and \
+        # default values are specified in the simulation script, so
+        # excluded here
+        (args.__dict__[argkey] is not None)
+       ):
+        if lfirst:
+            command +=' C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+        else:
+            command +=',C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+        lfirst=False
+
+print('Submitting array job: '+command)
+os.system(command)
diff --git a/class4gl/setup/batch_setup_global_old.py b/class4gl/setup/batch_setup_global_old.py
new file mode 100644
index 0000000..4a3f623
--- /dev/null
+++ b/class4gl/setup/batch_setup_global_old.py
@@ -0,0 +1,42 @@
+
+
+''' 
+Purpose: 
+    launch array job to get sounding and other global forcing data in class4gl input format"
+Usage:
+    python start_setup_global.py
+
+Author:
+    Hendrik Wouters 
+
+'''
+
+import pandas as pd
+import os
+import math
+import numpy as np
+import sys
+
+odir = "/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/GLOBAL/"
+fn_stations = odir+'/igra-stations_sel.txt'
+df_stations = pd.read_csv(fn_stations)
+
+# if sys.argv[1] == 'qsub':
+# with qsub
+STNlist = list(df_stations.iterrows())
+NUMSTNS = len(STNlist)
+PROCS = len(STNlist) 
+print(PROCS)
+BATCHSIZE = math.ceil(np.float(NUMSTNS)/np.float(PROCS))
+os.system('qsub /user/data/gent/gvo000/gvo00090/D2D/scripts/SOUNDINGS/setup_global.pbs -t 0-'+str(PROCS-1))
+# elif sys.argv[1] == 'wsub':
+#     
+#     
+#     # with wsub
+#     STNlist = list(df_stations.iterrows())
+#     NUMSTNS = len(STNlist)
+#     PROCS = NUMSTNS 
+#     BATCHSIZE = 1 #math.ceil(np.float(NUMSTNS)/np.float(PROCS))
+# 
+#     os.system('wsub -batch /user/data/gent/gvo000/gvo00090/D2D/scripts/SOUNDINGS/setup_global.pbs -t 0-'+str(PROCS-1))
+
diff --git a/class4gl/setup/batch_setup_igra.pbs b/class4gl/setup/batch_setup_igra.pbs
new file mode 100644
index 0000000..9777de5
--- /dev/null
+++ b/class4gl/setup/batch_setup_igra.pbs
@@ -0,0 +1,34 @@
+#!/bin/bash 
+#
+#PBS -j oe
+#PBS -M hendrik.wouters@ugent.be
+#PBS -m b
+#PBS -m e
+#PBS -m a
+#PBS -N c4gl_setup
+
+module purge
+source ~/.bashrc
+
+echo loading modules: $LOADDEPSCLASS4GL 
+$LOADDEPSCLASS4GL 
+
+EXEC_ALL="python $C4GLJOB_exec --first_station_row $PBS_ARRAYID --last_station_row $PBS_ARRAYID"
+
+for var in $(compgen -v | grep C4GLJOB_ ); do
+    echo $var
+    if [ "$var" != "C4GLJOB_exec" ]
+    then
+    EXEC_ALL=$EXEC_ALL" --"`echo $var | cut -c9-`"="${!var}
+    fi
+done
+
+
+# EXEC_ALL="python $exec --global-chunk-number $PBS_ARRAYID \
+#                        --split-by $split_by \
+#                        --dataset $dataset \
+#                        --experiments $experiments"
+#                  #      --path-soundings $path_soundings \
+echo Executing: $EXEC_ALL
+$EXEC_ALL
+
diff --git a/class4gl/setup/batch_setup_igra.py b/class4gl/setup/batch_setup_igra.py
new file mode 100644
index 0000000..4950dcf
--- /dev/null
+++ b/class4gl/setup/batch_setup_igra.py
@@ -0,0 +1,187 @@
+
+# -*- coding: utf-8 -*-
+
+import logging
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+import importlib
+spam_loader = importlib.find_loader('Pysolar')
+found = spam_loader is not None
+if found:
+    import Pysolar
+    import Pysolar.util.GetSunriseSunset
+else:
+    import pysolar as Pysolar
+    GetSunriseSunset =  Pysolar.util.get_sunrise_sunset
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--exec') # chunk simulation script
+parser.add_argument('--pbs_string',default=' -l walltime=70:00:00')
+parser.add_argument('--path_input')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+# parser.add_argument('--first_YYYYMMDD',default="19810101")
+# parser.add_argument('--last_YYYYMMDD',default="20180101")
+# parser.add_argument('--first_station_row')
+# parser.add_argument('--last_station_row')
+# parser.add_argument('--station_id') # run a specific station id
+# parser.add_argument('--latitude') # run a specific station id
+# parser.add_argument('--longitude') # run a specific station id
+# parser.add_argument('--error_handling',default='dump_on_success')
+# parser.add_argument('--subset_output',default='morning') # this tells which yaml subset
+                                                      # to initialize with.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+# arser.add_argument('--runtime',default='from_afternoon_profile')
+
+# parser.add_argument('--split_by',default="-1")# station soundings are split
+                                            # up in chunks
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+EXP_DEFS  =\
+{
+  'ERA-INTERIM_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'IOPS_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'IOPS_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+
+# iniitialize global data
+# ===============================
+print("Initializing global data")
+# ===============================
+globaldata = data_global()
+globaldata.sources = {**globaldata.sources,**{
+    
+        "ERAINT:t"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/t_6hourly/t_19830609-19830808_6hourly.nc",
+        "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q_19830609-19830808_6hourly.nc",
+        "ERAINT:u"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/u_6hourly/u_19830609-19830808_6hourly.nc",
+        "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v_19830609-19830808_6hourly.nc",
+    
+#        "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q_19830209-19830410_6hourly.nc",
+ #       "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q*_6hourly.nc",
+ #       "ERAINT:u"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/u_6hourly/u*_6hourly.nc",
+ #       "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v*_6hourly.nc",
+        }}
+
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+
+
+# ===============================
+print("getting a list of stations")
+# ===============================
+all_stations = stations(args.path_input,refetch_stations=False)
+
+
+# # ===============================
+# print("Selecting station by ID")
+# # ===============================
+# stations_iter = stations_iterator(all_stations)
+# STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+# all_stations_select = pd.DataFrame([run_station])
+# print(run_station)
+
+
+# ====================================
+print('defining all_stations_select')
+# ====================================
+
+#  # these are all the stations that are supposed to run by the whole batch (all
+#  # chunks). We narrow it down according to the station(s) specified.
+#  if (args.latitude is not None) or (args.longitude is not None):
+#      print('custom coordinates not implemented yet, please ask developer.')
+#  elif args.station_id is not None:
+#      print("Selecting station by ID")
+#      stations_iter = stations_iterator(all_stations)
+#      STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+#      all_stations_select = pd.DataFrame([run_station])
+#  #     print("making a custom station according to the coordinates")
+#  # 
+#  #     STNID = 43.23
+#  else:
+all_stations_select = pd.DataFrame(all_stations.table)
+#      if args.last_station_row is not None:
+#          all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+#      if args.first_station_row is not None:
+#          all_stations_select = all_station_select.iloc[int(args.first_station):]
+
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+# dtfirst = dt.datetime.strptime(args.first_YYYYMMDD,"%Y%m%d",)
+# dtlast = dt.datetime.strptime(args.last_YYYYMMDD,"%Y%m%d",)
+# # ===============================
+# print("Creating daily timeseries from", dtfirst," to ", dtlast)
+# # ===============================
+# DTS = [dtfirst + dt.timedelta(days=iday) for iday in \
+#        range(int((dtlast + dt.timedelta(days=1) -
+#                   dtfirst).total_seconds()/3600./24.))]
+# 
+# if int(args.split_by) != -1:
+#     totalchunks = len(all_stations_select)*math.ceil(len(DTS)/int(args.split_by))
+# else:
+totalchunks = len(all_stations_select)
+# 
+# print(totalchunks)
+
+#if args.cleanup_experiments:
+#    os.system("rm -R "+args.path_experiments+'/')
+
+# C4GLJOB_timestamp="+dt.datetime.now().isoformat()+",
+command = 'qsub '+args.pbs_string+' '+args.c4gl_path_lib+'/setup/batch_setup_igra.pbs -t 0-'+\
+            str(totalchunks-1)+" -v "
+# propagate arguments towards the job script
+lfirst = True
+for argkey in args.__dict__.keys():
+    if ((argkey not in ['experiments','pbs_string','cleanup_experiments']) and \
+        # default values are specified in the simulation script, so
+        # excluded here
+        (args.__dict__[argkey] is not None)
+       ):
+        print(argkey)
+        print(args.__dict__[argkey])
+        if lfirst:
+            command +=' C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+        else:
+            command +=',C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+        lfirst=False
+
+print('Submitting array job: '+command)
+os.system(command)
diff --git a/class4gl/setup/batch_update.py b/class4gl/setup/batch_update.py
new file mode 100644
index 0000000..ab51951
--- /dev/null
+++ b/class4gl/setup/batch_update.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+
+"""
+Usage:
+python batch_update.py --exec $CLASS4GL/simulations/update_yaml_old.py
+--path_experiments $VSC_DATA_VO/D2D/data/C4GL/GLOBAL_NOAC/ --path_input
+$VSC_DATA_VO/D2D/data/C4GL/GLOBAL_NOAC_BACKUP_20180904/ --c4gl_path_lib
+$CLASS4GL --split_by 50 --global_keys "KGC" --subset_input morning --experiments
+"GLOBAL_NOAC"
+"""
+
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+parser = argparse.ArgumentParser()
+#if __name__ == '__main__':
+parser.add_argument('--exec') # chunk simulation script
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--pbs_string',default=' -l walltime=2:0:0')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--error_handling')
+parser.add_argument('--subset_input',default='morning') 
+                                        # this tells which yaml subset
+                                        # to initialize with.
+                                        # Most common options are
+                                        # 'morning' and 'ini'.
+parser.add_argument('--subset_output',default='morning') 
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime')
+# delete folders of experiments before running them
+parser.add_argument('--split_by',default=50)# station soundings are split
+                                            # up in chunks
+
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--path_input') #,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output') #,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+
+
+#arguments only used for update_yaml.py
+parser.add_argument('--path_dataset') 
+parser.add_argument('--global_keys') 
+parser.add_argument('--updates') 
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+print("getting all stations from --path_input")
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_input,suffix=args.subset_input,refetch_stations=False)
+
+print('defining all_stations_select')
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+if args.station_id is not None:
+    print("Selecting stations by --station_id")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table [--first_station_row,--last_station_row]")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         args.path_input,\
+                                         subset=args.subset_input,\
+                                         refetch_records=False,\
+                                        )
+
+print('splitting batch in --split_by='+args.split_by+' jobs.')
+totalchunks = 0
+for istation,current_station in all_stations_select.iterrows():
+    records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+    chunks_current_station = len(records_morning_station_select.query('STNID == '+str(current_station.name)).chunk.unique())
+    totalchunks +=chunks_current_station
+
+print('total chunks (= size of array-job) per experiment: ' + str(totalchunks))
+
+#if sys.argv[1] == 'qsub':
+# with qsub
+
+
+
+#C4GLJOB_timestamp="+dt.datetime.now().isoformat()+",
+command = 'qsub '+args.pbs_string+' '+args.c4gl_path_lib+'/simulations/batch_simulations.pbs -t 0-'+\
+            str(totalchunks-1)+" -v '"
+# propagate arguments towards the job script
+first = True
+for argkey in args.__dict__.keys():
+    if ((argkey not in ['pbs_string']) and \
+        # default values are specified in the simulation script, so
+        # excluded here
+        (args.__dict__[argkey] is not None)
+       ):
+        if first:
+            command +='C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+        else:
+            command +=',C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+    first = False
+
+command = command+"'"
+print('Submitting array job: '+command)
+os.system(command)
+
+
+    #os.system(command)
+# elif sys.argv[1] == 'wsub':
+#     
+#     # with wsub
+#     STNlist = list(df_stations.iterrows())
+#     NUMSTNS = len(STNlist)
+#     PROCS = NUMSTNS 
+#     BATCHSIZE = 1 #math.ceil(np.float(NUMSTNS)/np.float(PROCS))
+# 
+#     os.system('wsub -batch /user/data/gent/gvo000/gvo00090/D2D/scripts/C4GL/global_run.pbs -t 0-'+str(PROCS-1))
+
diff --git a/class4gl/setup/batch_update_input.py b/class4gl/setup/batch_update_input.py
new file mode 100644
index 0000000..355f532
--- /dev/null
+++ b/class4gl/setup/batch_update_input.py
@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+parser = argparse.ArgumentParser()
+#if __name__ == '__main__':
+parser.add_argument('--exec') # chunk simulation script
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--pbs_string',default=' -l walltime=2:0:0')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--error_handling')
+parser.add_argument('--updates')
+parser.add_argument('--global_vars')
+parser.add_argument('--subset_input',default='morning') 
+                                        # this tells which yaml subset
+                                        # to initialize with.
+                                        # Most common options are
+                                        # 'morning' and 'ini'.
+parser.add_argument('--subset_output',default='morning') 
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime')
+# delete folders of experiments before running them
+parser.add_argument('--cleanup_output',default=False)
+parser.add_argument('--split_by',default=50)# station soundings are split
+                                            # up in chunks
+
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--path_input') #,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output') #,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+
+
+#arguments only used for update_yaml.py
+parser.add_argument('--path_dataset') 
+parser.add_argument('--global_keys') 
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+
+# #SET = 'GLOBAL'
+# SET = args.dataset
+
+# path_inputSET = args.path_input+'/'+SET+'/'
+
+print("getting all stations from --path_input")
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_input,suffix=args.subset_input,refetch_stations=False)
+
+print('defining all_stations_select')
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+if args.station_id is not None:
+    print("Selecting stations by --station_id")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table [--first_station_row,--last_station_row]")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         args.path_input,\
+                                         subset=args.subset_input,\
+                                         refetch_records=False,\
+                                        )
+
+print('splitting batch in --split_by='+args.split_by+' jobs.')
+totalchunks = 0
+for istation,current_station in all_stations_select.iterrows():
+    records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+    chunks_current_station = math.ceil(float(len(records_morning_station_select))/float(args.split_by))
+    totalchunks +=chunks_current_station
+
+print('total chunks (= size of array-job): ' + str(totalchunks))
+
+#if sys.argv[1] == 'qsub':
+# with qsub
+
+
+if args.cleanup_output:
+    os.system("rm -R "+args.path_output+'/')
+
+# C4GLJOB_timestamp="+dt.datetime.now().isoformat()+",
+command = 'qsub '+args.pbs_string+' '+args.c4gl_path_lib+'/simulations/batch_simulations.pbs -t 0-'+\
+            str(totalchunks-1)+" -v '"
+# propagate arguments towards the job script
+first = True
+for argkey in args.__dict__.keys():
+    if ((argkey not in ['pbs_string','cleanup_output']) and \
+        # default values are specified in the simulation script, so
+        # excluded here
+        (args.__dict__[argkey] is not None)
+       ):
+        if first:
+            command +='C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+        else:
+            command +=',C4GLJOB_'+argkey+'='+args.__dict__[argkey]
+    first = False
+
+command = command+"'"
+print('Submitting array job: '+command)
+os.system(command)
+
+
+    #os.system(command)
+# elif sys.argv[1] == 'wsub':
+#     
+#     # with wsub
+#     STNlist = list(df_stations.iterrows())
+#     NUMSTNS = len(STNlist)
+#     PROCS = NUMSTNS 
+#     BATCHSIZE = 1 #math.ceil(np.float(NUMSTNS)/np.float(PROCS))
+# 
+#     os.system('wsub -batch /user/data/gent/gvo000/gvo00090/D2D/scripts/C4GL/global_run.pbs -t 0-'+str(PROCS-1))
+
diff --git a/class4gl/setup/setup_bllast.py b/class4gl/setup/setup_bllast.py
new file mode 100644
index 0000000..bf779d6
--- /dev/null
+++ b/class4gl/setup/setup_bllast.py
@@ -0,0 +1,727 @@
+# -*- coding: utf-8 -*-
+# Read data from BLLAST campaing and convert it to class4gl input
+
+# WARNING!! stupid tab versus space formatting, grrrmmmlmlmlll!  the following command needs to be executed first: 
+#    for file in RS_2011????_????_site1_MODEM_CRA.cor ;  do expand -i -t 4 $file > $file.fmt ; done
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import Pysolar
+import sys
+import pytz
+sys.path.insert(0,'/user/home/gent/vsc422/vsc42247/software/class4gl/class4gl/')
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+
+globaldata = data_global()
+globaldata.load_datasets(recalc=0)
+
+Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+epsilon = Rd/Rv # or mv/md
+
+
+def replace_iter(iterable, search, replace):
+    for value in iterable:
+        value.replace(search, replace)
+        yield value
+
+from class4gl import blh,class4gl_input
+
+# definition of the humpa station
+current_station = pd.Series({ "latitude"  : 42.971834,
+                  "longitude" : 0.3671169,
+                  "name" : "the BLLAST experiment"
+                })
+current_station.name = 90001
+
+
+
+
+
+# RS_20110624_1700_site1_MODEM_CRA.cor.fmt
+# RS_20110630_1700_site1_MODEM_CRA.cor.fmt
+# RS_20110702_1655_site1_MODEM_CRA.cor.fmt
+# RS_20110621_0509_site1_MODEM_CRA.cor.fmt
+
+HOUR_FILES = \
+{ dt.datetime(2011,6,19,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110619_0521_site1_MODEM_CRA.cor.fmt'],'afternoon':[18,'RS_20110619_1750_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,6,20,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110620_0515_site1_MODEM_CRA.cor.fmt'],'afternoon':[18,'RS_20110620_1750_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,6,25,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110625_0500_site1_MODEM_CRA.cor.fmt'],'afternoon':[17,'RS_20110625_1700_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,6,26,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110626_0500_site1_MODEM_CRA.cor.fmt'],'afternoon':[17,'RS_20110626_1700_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,6,27,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110627_0503_site1_MODEM_CRA.cor.fmt'],'afternoon':[17,'RS_20110627_1700_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,7, 2,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110702_0501_site1_MODEM_CRA.cor.fmt'],'afternoon':[17,'RS_20110702_1655_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,7, 5,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110705_0448_site1_MODEM_CRA.cor.fmt'],'afternoon':[17,'RS_20110705_1701_site1_MODEM_CRA.cor.fmt']},
+}
+
+
+#only include the following timeseries in the model output
+timeseries_only = \
+['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+ 'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+ 'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+ 'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+ 'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+def esat(T):
+    return 0.611e3 * np.exp(17.2694 * (T - 273.16) / (T - 35.86))
+def efrom_rh100_T(rh100,T):
+    return esat(T)*rh100/100.
+def qfrom_e_p(e,p):
+    return epsilon * e/(p - (1.-epsilon)*e)
+
+def bllast_parser(balloon_file,file_sounding,ldate,hour,c4gli=None):
+        #balloon_conv = replace_iter(balloon_file,"°","deg")
+        #readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()]
+        #air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15)
+        air_balloon_in = pd.read_csv(balloon_file,delimiter='\t',)
+                                     #widths=[14]*19,
+                                     #skiprows=9,
+                                     #skipfooter=15,
+                                     #decimal='.',
+                                     #header=None,
+                                     #names = columns,
+                                     #na_values='-----')
+        air_balloon_in = air_balloon_in.rename(columns=lambda x: x.strip())
+        print(air_balloon_in.columns)
+        rowmatches = {
+            't':      lambda x: x['TaRad']+273.15,
+            #'tv':     lambda x: x['Virt. Temp[C]']+273.15,
+            'p':      lambda x: x['Press']*100.,
+            'u':      lambda x: x['VHor'] * np.sin((90.-x['VDir'])/180.*np.pi),
+            'v':      lambda x: x['VHor'] * np.cos((90.-x['VDir'])/180.*np.pi),
+            'z':      lambda x: x['Altitude'] -582.,
+            # from virtual temperature to absolute humidity
+            'q':      lambda x: qfrom_e_p(efrom_rh100_T(x['UCal'],x['TaRad']+273.15),x['Press']*100.),
+        }
+        
+        air_balloon = pd.DataFrame()
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon_in)
+        
+        rowmatches = {
+            'R' :    lambda x: (Rd*(1.-x.q) + Rv*x.q),
+            'theta': lambda x: (x['t']) * (x['p'][0]/x['p'])**(x['R']/cp),
+            'thetav': lambda x: x.theta  + 0.61 * x.theta * x.q
+        }
+        
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon)
+        
+        dpars = {}
+        dpars['longitude']  = current_station['longitude']
+        dpars['latitude']  = current_station['latitude'] 
+        
+        dpars['STNID'] = current_station.name
+        
+        
+        is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)
+        valid_indices = air_balloon.index[is_valid].values
+        i = 1
+        while (air_balloon.thetav.iloc[valid_indices[0]] - \
+               air_balloon.thetav.iloc[valid_indices[i]] ) > 0.5:
+            #diff = (air_balloon.theta.iloc[valid_indices[i]] -air_balloon.theta.iloc[valid_indices[i+1]])- 0.5
+            air_balloon.thetav.iloc[valid_indices[0:i]] = \
+                air_balloon.thetav.iloc[valid_indices[i]] + 0.5 
+            
+            i +=1
+        
+        air_ap_mode='b'
+        
+        if len(valid_indices) > 0:
+            dpars['h'],dpars['h_u'],dpars['h_l'] =\
+                blh(air_balloon.z,air_balloon.thetav,air_balloon_in['VHor'])
+            dpars['h_b'] = np.max((dpars['h'],10.))
+            dpars['h_u'] = np.max((dpars['h_u'],10.)) #upper limit of mixed layer height
+            dpars['h_l'] = np.max((dpars['h_l'],10.)) #low limit of mixed layer height
+            dpars['h_e'] = np.abs( dpars['h_u'] - dpars['h_l']) # error of mixed-layer height
+            dpars['h'] = np.round(dpars['h_'+air_ap_mode],1)
+        else:
+            dpars['h_u'] =np.nan
+            dpars['h_l'] =np.nan
+            dpars['h_e'] =np.nan
+            dpars['h'] =np.nan
+        
+        
+        
+        if ~np.isnan(dpars['h']):
+            dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]]
+        else:
+            dpars['Ps'] = np.nan
+        
+        if ~np.isnan(dpars['h']):
+        
+            # determine mixed-layer properties (moisture, potential temperature...) from profile
+            
+            # ... and those of the mixed layer
+            is_valid_below_h = (air_balloon.iloc[valid_indices].z < dpars['h'])
+            valid_indices_below_h =  air_balloon.iloc[valid_indices].index[is_valid_below_h].values
+            if len(valid_indices) > 1:
+                if len(valid_indices_below_h) >= 3.:
+                    ml_mean = air_balloon.iloc[valid_indices][is_valid_below_h].mean()
+                else:
+                    ml_mean = air_balloon.iloc[valid_indices[0]:valid_indices[1]].mean()
+            elif len(valid_indices) == 1:
+                ml_mean = (air_balloon.iloc[0:1]).mean()
+            else:
+                temp =  pd.DataFrame(air_balloon)
+                temp.iloc[0] = np.nan
+                ml_mean = temp
+                       
+            dpars['theta']= ml_mean.theta
+            dpars['q']    = ml_mean.q
+            dpars['u']    = ml_mean.u
+            dpars['v']    = ml_mean.v 
+        else:
+            dpars['theta'] = np.nan
+            dpars['q'] = np.nan
+            dpars['u'] = np.nan
+            dpars['v'] = np.nan
+        
+        air_ap_head = air_balloon[0:0] #pd.DataFrame(columns = air_balloon.columns)
+        # All other  data points above the mixed-layer fit
+        air_ap_tail = air_balloon[air_balloon.z > dpars['h']]
+
+
+
+
+
+        air_ap_head.z = pd.Series(np.array([2.,dpars['h'],dpars['h']]))
+        jump = air_ap_head.iloc[0] * np.nan
+        
+        
+        if air_ap_tail.shape[0] > 1:
+        
+            # we originally used THTA, but that has another definition than the
+            # variable theta that we need which should be the temperature that
+            # one would have if brought to surface (NOT reference) pressure.
+            for column in ['theta','q','u','v']:
+               
+               # initialize the profile head with the mixed-layer values
+               air_ap_head[column] = ml_mean[column]
+               # calculate jump values at mixed-layer height, which will be
+               # added to the third datapoint of the profile head
+               jump[column] = (air_ap_tail[column].iloc[1]\
+                               -\
+                               air_ap_tail[column].iloc[0])\
+                              /\
+                              (air_ap_tail.z.iloc[1]\
+                               - air_ap_tail.z.iloc[0])\
+                              *\
+                              (dpars['h']- air_ap_tail.z.iloc[0])\
+                              +\
+                              air_ap_tail[column].iloc[0]\
+                              -\
+                              ml_mean[column] 
+               if column == 'theta':
+                  # for potential temperature, we need to set a lower limit to
+                  # avoid the model to crash
+                  jump.theta = np.max((0.1,jump.theta))
+        
+               air_ap_head[column][2] += jump[column]
+        
+        air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 +air_ap_head.v**2)
+
+
+        # filter data so that potential temperature always increases with
+        # height 
+        cols = []
+        for column in air_ap_tail.columns:
+            #if column != 'z':
+                cols.append(column)
+
+        # only select samples monotonically increasing with height
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        for ibottom in range(1,len(air_ap_tail_orig)):
+            if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z +10.:
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],ignore_index=True)
+
+
+
+
+        # make theta increase strong enough to avoid numerical
+        # instability
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        theta_low = air_ap_head['theta'].iloc[2]
+        z_low = air_ap_head['z'].iloc[2]
+        ibottom = 0
+        for itop in range(0,len(air_ap_tail_orig)):
+            theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+            z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+            if (
+                #(z_mean > z_low) and \
+                (z_mean > (z_low+10.)) and \
+                #(theta_mean > (theta_low+0.2) ) and \
+                #(theta_mean > (theta_low+0.2) ) and \
+                 (((theta_mean - theta_low)/(z_mean - z_low)) > 0.0001)):
+
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+                ibottom = itop+1
+                theta_low = air_ap_tail.theta.iloc[-1]
+                z_low =     air_ap_tail.z.iloc[-1]
+            # elif  (itop > len(air_ap_tail_orig)-10):
+            #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        
+        air_ap = \
+            pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+
+
+
+        # # make theta increase strong enough to avoid numerical
+        # # instability
+        # air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        # air_ap_tail = pd.DataFrame()
+        # #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # theta_low = air_ap_head['theta'].iloc[2]
+        # z_low = air_ap_head['z'].iloc[2]
+        # ibottom = 0
+        # for itop in range(0,len(air_ap_tail_orig)):
+        #     theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+        #     z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+        #     if ((theta_mean > (theta_low+0.2) ) and \
+        #          (((theta_mean - theta_low)/(z_mean - z_low)) > 0.001)):
+
+        #         air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+        #         ibottom = itop+1
+        #         theta_low = air_ap_tail.theta.iloc[-1]
+        #         z_low =     air_ap_tail.z.iloc[-1]
+        #     # elif  (itop > len(air_ap_tail_orig)-10):
+        #     #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        # air_ap = \
+        #     pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+        # 
+        # # we copy the pressure at ground level from balloon sounding. The
+        # # pressure at mixed-layer height will be determined internally by class
+        
+        rho        = 1.2                   # density of air [kg m-3]
+        g          = 9.81                  # gravity acceleration [m s-2]
+        
+        air_ap['p'].iloc[0] =dpars['Ps'] 
+        air_ap['p'].iloc[1] =(dpars['Ps'] - rho * g * dpars['h'])
+        air_ap['p'].iloc[2] =(dpars['Ps'] - rho * g * dpars['h'] -0.1)
+        
+        
+        dpars['lat'] = dpars['latitude']
+        # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich)
+        dpars['lon'] = 0.
+        # this is the real longitude that will be used to extract ground data
+        
+        dpars['ldatetime'] = ldate+dt.timedelta(hours=hour)
+        dpars['datetime'] = ldate+dt.timedelta(hours=hour)
+        dpars['doy'] = dpars['datetime'].timetuple().tm_yday
+        
+        dpars['SolarAltitude'] = \
+                                Pysolar.GetAltitude(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        dpars['SolarAzimuth'] =  Pysolar.GetAzimuth(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        
+        
+        dpars['lSunrise'], dpars['lSunset'] \
+        =  Pysolar.util.GetSunriseSunset(dpars['latitude'],
+                                         0.,
+                                         dpars['ldatetime'],0.)
+        
+        # Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually
+        # write local solar time, we need to assign the timezone to UTC (which
+        # is WRONG!!!). Otherwise ruby cannot understand it (it always converts
+        # tolocal computer time :( ). 
+        dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise'])
+        dpars['lSunset'] = pytz.utc.localize(dpars['lSunset'])
+        
+        # This is the nearest datetime when the sun is up (for class)
+        dpars['ldatetime_daylight'] = \
+                                np.min(\
+                                    (np.max(\
+                                        (dpars['ldatetime'],\
+                                         dpars['lSunrise']+dt.timedelta(hours=2))\
+                                     ),\
+                                     dpars['lSunset']\
+                                    )\
+                                )
+        # apply the same time shift for UTC datetime
+        dpars['datetime_daylight'] = dpars['datetime'] \
+                                    +\
+                                    (dpars['ldatetime_daylight']\
+                                     -\
+                                     dpars['ldatetime'])
+        
+        print('ldatetime_daylight',dpars['ldatetime_daylight'])
+        print('ldatetime',dpars['ldatetime'])
+        print('lSunrise',dpars['lSunrise'])
+        dpars['day'] = dpars['ldatetime'].day
+        
+        # We set the starting time to the local sun time, since the model 
+        # thinks we are always at the meridian (lon=0). This way the solar
+        # radiation is calculated correctly.
+        dpars['tstart'] = dpars['ldatetime_daylight'].hour \
+                         + \
+                         dpars['ldatetime_daylight'].minute/60.\
+                         + \
+                         dpars['ldatetime_daylight'].second/3600.
+        
+        print('tstart',dpars['tstart'])
+        dpars['sw_lit'] = False
+        # convert numpy types to native python data types. This provides
+        # cleaner data IO with yaml:
+        for key,value in dpars.items():
+            if type(value).__module__ == 'numpy':
+                dpars[key] = dpars[key].item()
+        
+                decimals = {'p':0,'t':2,'theta':4, 'z':2, 'q':5, 'u':4, 'v':4}
+        # 
+                for column,decimal in decimals.items():
+                    air_balloon[column] = air_balloon[column].round(decimal)
+                    air_ap[column] = air_ap[column].round(decimal)
+        
+        updateglobal = False
+        if c4gli is None:
+            c4gli = class4gl_input()
+            updateglobal = True
+        
+        print('updating...')
+        print(column)
+        c4gli.update(source='bllast',\
+                    # pars=pars,
+                    pars=dpars,\
+                    air_balloon=air_balloon,\
+                    air_ap=air_ap)
+        if updateglobal:
+            c4gli.get_global_input(globaldata)
+
+        # if profile_ini:
+        #     c4gli.runtime = 10 * 3600
+
+        c4gli.dump(file_sounding)
+        
+        # if profile_ini:
+        #     c4gl = class4gl(c4gli)
+        #     c4gl.run()
+        #     c4gl.dump(file_model,\
+        #               include_input=True,\
+        #               timeseries_only=timeseries_only)
+        #     
+        #     # This will cash the observations and model tables per station for
+        #     # the interface
+        # 
+        # if profile_ini:
+        #     profile_ini=False
+        # else:
+        #     profile_ini=True
+        return c4gli
+
+
+path_soundings = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/IOPS/'
+
+
+file_morning = open(path_soundings+format(current_station.name,'05d')+'_ini.yaml','w') 
+for date,pair  in HOUR_FILES.items(): 
+    print(pair['morning'])
+    humpafn ='/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/BLLAST/MODEM Radiosoundings/'+pair['morning'][1]
+    
+    print(humpafn)
+    balloon_file = open(humpafn,'r',encoding='latin-1')
+
+    c4gli_morning = bllast_parser(balloon_file,file_morning,date,pair['morning'][0])
+    print('c4gli_morning_ldatetime 0',c4gli_morning.pars.ldatetime)
+file_morning.close()
+
+file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_end.yaml','w') 
+for date,pair  in HOUR_FILES.items(): 
+    humpafn ='/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/BLLAST/MODEM Radiosoundings/'+pair['afternoon'][1]
+    balloon_file = open(humpafn,'r',encoding='latin-1')
+
+    c4gli_afternoon = bllast_parser(balloon_file,file_afternoon,date,pair['afternoon'][0])
+    print('c4gli_afternoon_ldatetime 0',c4gli_afternoon.pars.ldatetime)
+file_afternoon.close()
+ 
+
+# file_morning = open(path_soundings+format(current_station.name,'05d')+'_morning.yaml','w') 
+# for date,pair  in HOUR_FILES.items(): 
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/'+pair['morning'][1],
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_morning,hour,c4gli_morning)
+#     print('c4gli_morning_ldatetime 1',c4gli_morning.pars.ldatetime)
+# file_morning.close()
+# 
+# file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_afternoon.yaml','w') 
+# for hour in [18]:
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/humppa_080610_'+format(hour,"02d")+'00.txt'
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_afternoon,hour,c4gli_afternoon)
+# file_afternoon.close()
+
+
+
+# path_model = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/HUMPPA/'
+# 
+# file_model    = open(fnout_model+   format(current_station.name,'05d')+'.yaml','w') 
+
+
+records_morning = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='ini',
+                                           refetch_records=True,
+                                           )
+print('records_morning_ldatetime',records_morning.ldatetime)
+
+records_afternoon = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='end',
+                                           refetch_records=True,
+                                           )
+
+# # align afternoon records with noon records, and set same index
+# records_afternoon.index = records_afternoon.ldatetime.dt.date
+# records_afternoon = records_afternoon.loc[records_morning.ldatetime.dt.date]
+# records_afternoon.index = records_morning.index
+# path_exp = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/IOPS/'
+# 
+# os.system('mkdir -p '+path_exp)
+# file_morning = open(path_soundings+'/'+format(current_station.name,'05d')+'_morning.yaml')
+# file_afternoon = open(path_soundings+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+# file_ini = open(path_exp+'/'+format(current_station.name,'05d')+'_ini.yaml','w')
+# file_mod = open(path_exp+'/'+format(current_station.name,'05d')+'_mod.yaml','w')
+# 
+# for (STNID,chunk,index),record_morning in records_morning.iterrows():
+#     record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+# 
+#     c4gli_morning = get_record_yaml(file_morning, 
+#                                     record_morning.index_start, 
+#                                     record_morning.index_end,
+#                                     mode='ini')
+#     #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+#     
+#     
+#     c4gli_afternoon = get_record_yaml(file_afternoon, 
+#                                       record_afternoon.index_start, 
+#                                       record_afternoon.index_end,
+#                                     mode='ini')
+# 
+#     c4gli_morning.update(source='pairs',pars={'runtime' : \
+#                         int((c4gli_afternoon.pars.datetime_daylight - 
+#                              c4gli_morning.pars.datetime_daylight).total_seconds())})
+# 
+#     
+#     c4gli_morning.pars.sw_ac = []
+#     c4gli_morning.pars.sw_ap = True
+#     c4gli_morning.pars.sw_lit = False
+#     c4gli_morning.dump(file_ini)
+#     
+#     c4gl = class4gl(c4gli_morning)
+#     c4gl.run()
+#     
+#     c4gl.dump(file_mod,\
+#               include_input=False,\
+#               timeseries_only=timeseries_only)
+# file_ini.close()
+# file_mod.close()
+# file_morning.close()
+# file_afternoon.close()
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                            path_exp,\
+#                                            subset='ini',
+#                                            refetch_records=True,
+#                                            )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                            path_exp,\
+#                                            subset='mod',
+#                                            refetch_records=True,
+#                                            )
+# 
+# records_mod.index = records_ini.index
+# 
+# # align afternoon records with initial records, and set same index
+# records_afternoon.index = records_afternoon.ldatetime.dt.date
+# records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+# records_afternoon.index = records_ini.index
+
+
+
+# stations_for_iter = stations(path_exp)
+# for STNID,station in stations_iterator(stations_for_iter):
+#     records_current_station_index = \
+#             (records_ini.index.get_level_values('STNID') == STNID)
+#     file_current_station_mod = STNID
+# 
+#     with \
+#     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+#     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+#     open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+#         for (STNID,index),record_ini in records_iterator(records_ini):
+#             c4gli_ini = get_record_yaml(file_station_ini, 
+#                                         record_ini.index_start, 
+#                                         record_ini.index_end,
+#                                         mode='ini')
+#             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+# 
+#             record_mod = records_mod.loc[(STNID,index)]
+#             c4gl_mod = get_record_yaml(file_station_mod, 
+#                                         record_mod.index_start, 
+#                                         record_mod.index_end,
+#                                         mode='mod')
+#             record_afternoon = records_afternoon.loc[(STNID,index)]
+#             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+#                                         record_afternoon.index_start, 
+#                                         record_afternoon.index_end,
+#                                         mode='ini')
+
+
+# # select the samples of the afternoon list that correspond to the timing of the
+# # morning list
+# records_afternoon = records_afternoon.set_index('ldatetime').loc[records_afternoon.ldatetime)]
+# records_afternoon.index = recods_morning.index
+# 
+# 
+# # create intersectino index
+# index_morning = pd.Index(records_morning.ldatetime.to_date())
+# index_afternoon = pd.Index(records_afternoon.ldatetime.to_date())
+# 
+# for record_morning in records_morning.iterrows():
+#     
+#     c4gl = class4gl(c4gli)
+#     c4gl.run()
+#     c4gl.dump(c4glfile,\
+#               include_input=True,\
+#               timeseries_only=timeseries_only)
+# 
+# # This will cash the observations and model tables per station for
+# # the interface
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=0,\
+#                                    by=2,\
+#                                    subset='ini',
+#                                    refetch_records=True,
+#                                    )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='mod',
+#                                    refetch_records=True,
+#                                    )
+# records_eval = get_records(pd.DataFrame([current_station]),\
+#                                    path_obs,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='eval',
+#                                    refetch_records=True,
+#                                    )
+# 
+# 
+# # mod_scores = pd.DataFrame(index=mod_records.index)
+# # for (STNID,index), current_record_mod in mod_records.iterrows():
+# #     print(STNID,index)
+# #     current_station = STN
+# #     current_record_obs_afternoon = obs_records_afternoon.loc[(STNID,index)]
+# #     current_record_obs = obs_records.loc[(STNID,index)]
+# # 
+# #     record_yaml_mod = get_record_yaml_mod(odirexperiments[keyEXP],\
+# #                                           current_station,\
+# #                                           current_record_mod,\
+# #                                          )
+# # 
+# #     record_yaml_obs = \
+# #             get_record_yaml_obs(odirexperiments[keyEXP],\
+# #                                 current_station,\
+# #                                 current_record_obs,\
+# #                                 suffix='.yaml')
+# # 
+# #     record_yaml_obs_afternoon = \
+# #             get_record_yaml_obs(odir,\
+# #                                 current_station,\
+# #                                 current_record_obs_afternoon,\
+# #                                 suffix='_afternoon.yaml')
+# # 
+# #     hmax = np.max([record_yaml_obs_afternoon.pars.h,\
+# #                    record_yaml_mod.h])
+# #     HEIGHTS = {'h':hmax, '2h':2.*hmax, '3000m':3000.}
+# #     
+# # 
+# #     for height,hvalue in HEIGHTS.items():
+# # 
+# #         lt_obs = (record_yaml_obs_afternoon.air_ap.HAGL < hvalue)
+# #         lt_mod = (record_yaml_mod.air_ap.z < hvalue)
+# #         try:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = \
+# #                 rmse(\
+# #                     record_yaml_obs_afternoon.air_ap.theta[lt_obs],\
+# #                     np.interp(\
+# #                         record_yaml_obs_afternoon.air_ap.HAGL[lt_obs],\
+# #                         record_yaml_mod.air_ap.z[lt_mod],\
+# #                         record_yaml_mod.air_ap.theta[lt_mod]\
+# #                     ))
+# #         except ValueError:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = np.nan
+# #     # # we calculate these things in the interface itself
+# #     # for key in ['q','theta','h']:
+# #     #     mod_records.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_mod.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# #     #     # the actual time of the initial and evaluation sounding can be 
+# #     #     # different, but we consider this as a measurement error for
+# #     #     # the starting and end time of the simulation.
+# #     #     obs_records_afternoon.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_obs_afternoon.pars.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# # mod_scores.to_pickle(odirexperiments[keyEXP]+'/'+format(STNID,'05d')+"_mod_scores.pkl")
+# #         
+# #                 
+# #                 
+# # # for EXP,c4glfile in c4glfiles.items():
+# # #     c4glfile.close()            
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# #     
+# #     # {'Time[min:sec]': None 
+# #     #  'P[hPa]': None, 
+# #     #  'T[C]': None, 
+# #     #  'U[%]': None, 
+# #     #  'Wsp[m/s]': None, 
+# #     #  'Wdir[Grd]': None,
+# #     #  'Lon[°]', 
+# #     #  'Lat[°]', 
+# #     #  'Altitude[m]', 'GeoPot[m']', 'MRI',
+# #     #        'Unnamed: 11', 'RI', 'Unnamed: 13', 'DewPoint[C]', 'Virt. Temp[C]',
+# #     #        'Rs[m/min]D[kg/m3]Azimut[deg]', 'Elevation[deg]', 'Range[m]']
+# #     # }
+# #     # 
+# #     # #pivotrows =
+# #     # #{
+# # 
+# # 
+# # 
diff --git a/class4gl/setup/setup_bllast_noon.py b/class4gl/setup/setup_bllast_noon.py
new file mode 100644
index 0000000..e846d25
--- /dev/null
+++ b/class4gl/setup/setup_bllast_noon.py
@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+# Read data from BLLAST campaing and convert it to class4gl input
+
+# WARNING!! stupid tab versus space formatting, grrrmmmlmlmlll!  the following command needs to be executed first: 
+#    for file in RS_2011????_????_site1_MODEM_CRA.cor ;  do expand -i -t 4 $file > $file.fmt ; done
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import Pysolar
+import sys
+import pytz
+sys.path.insert(0,'/user/home/gent/vsc422/vsc42247/software/class4gl/class4gl/')
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+
+globaldata = data_global()
+globaldata.load_datasets(recalc=0)
+
+Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+epsilon = Rd/Rv # or mv/md
+
+
+def replace_iter(iterable, search, replace):
+    for value in iterable:
+        value.replace(search, replace)
+        yield value
+
+from class4gl import blh,class4gl_input
+
+# definition of the humpa station
+current_station = pd.Series({ "latitude"  : 42.971834,
+                  "longitude" : 0.3671169,
+                  "name" : "the BLLAST experiment"
+                })
+current_station.name = 90001
+
+
+
+
+
+# RS_20110624_1700_site1_MODEM_CRA.cor.fmt
+# RS_20110630_1700_site1_MODEM_CRA.cor.fmt
+# RS_20110702_1655_site1_MODEM_CRA.cor.fmt
+# RS_20110621_0509_site1_MODEM_CRA.cor.fmt
+
+HOUR_FILES = \
+{ dt.datetime(2011,6,19,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110619_0521_site1_MODEM_CRA.cor.fmt'],'afternoon':[11.25,'RS_20110619_1115_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,6,20,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110620_0515_site1_MODEM_CRA.cor.fmt'],'afternoon':[11.25,'RS_20110620_1115_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,6,25,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110625_0500_site1_MODEM_CRA.cor.fmt'],'afternoon':[11,'RS_20110625_1100_site1_MODEM_CRA.cor.fmt']},
+# dt.datetime(2011,6,26,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110626_0500_site1_MODEM_CRA.cor.fmt'],'afternoon':[17,'RS_20110626_1700_site1_MODEM_CRA.cor.fmt']},
+# dt.datetime(2011,6,27,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110627_0503_site1_MODEM_CRA.cor.fmt'],'afternoon':[17,'RS_20110627_1700_site1_MODEM_CRA.cor.fmt']},
+ dt.datetime(2011,7, 2,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110702_0501_site1_MODEM_CRA.cor.fmt'],'afternoon':[11,'RS_20110702_1057_site1_MODEM_CRA.cor.fmt']},
+# dt.datetime(2011,7, 5,0,0,0,0,pytz.UTC):{'morning':[5,'RS_20110705_0448_site1_MODEM_CRA.cor.fmt'],'afternoon':[17,'RS_20110705_1701_site1_MODEM_CRA.cor.fmt']},
+}
+
+
+#only include the following timeseries in the model output
+timeseries_only = \
+['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+ 'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+ 'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+ 'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+ 'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+def esat(T):
+    return 0.611e3 * np.exp(17.2694 * (T - 273.16) / (T - 35.86))
+def efrom_rh100_T(rh100,T):
+    return esat(T)*rh100/100.
+def qfrom_e_p(e,p):
+    return epsilon * e/(p - (1.-epsilon)*e)
+
+def bllast_parser(balloon_file,file_sounding,ldate,hour,c4gli=None):
+        #balloon_conv = replace_iter(balloon_file,"°","deg")
+        #readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()]
+        #air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15)
+        air_balloon_in = pd.read_csv(balloon_file,delimiter='\t',)
+                                     #widths=[14]*19,
+                                     #skiprows=9,
+                                     #skipfooter=15,
+                                     #decimal='.',
+                                     #header=None,
+                                     #names = columns,
+                                     #na_values='-----')
+        air_balloon_in = air_balloon_in.rename(columns=lambda x: x.strip())
+        print(air_balloon_in.columns)
+        rowmatches = {
+            't':      lambda x: x['TaRad']+273.15,
+            #'tv':     lambda x: x['Virt. Temp[C]']+273.15,
+            'p':      lambda x: x['Press']*100.,
+            'u':      lambda x: x['VHor'] * np.sin((90.-x['VDir'])/180.*np.pi),
+            'v':      lambda x: x['VHor'] * np.cos((90.-x['VDir'])/180.*np.pi),
+            'z':      lambda x: x['Altitude'] -582.,
+            # from virtual temperature to absolute humidity
+            'q':      lambda x: qfrom_e_p(efrom_rh100_T(x['UCal'],x['TaRad']+273.15),x['Press']*100.),
+        }
+        
+        air_balloon = pd.DataFrame()
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon_in)
+        
+        rowmatches = {
+            'R' :    lambda x: (Rd*(1.-x.q) + Rv*x.q),
+            'theta': lambda x: (x['t']) * (x['p'][0]/x['p'])**(x['R']/cp),
+            'thetav': lambda x: x.theta  + 0.61 * x.theta * x.q
+        }
+        
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon)
+        
+        dpars = {}
+        dpars['longitude']  = current_station['longitude']
+        dpars['latitude']  = current_station['latitude'] 
+        
+        dpars['STNID'] = current_station.name
+        
+        
+        is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)
+        valid_indices = air_balloon.index[is_valid].values
+        
+        air_ap_mode='b'
+        
+        if len(valid_indices) > 0:
+            dpars['h'],dpars['h_u'],dpars['h_l'] =\
+                blh(air_balloon.z,air_balloon.thetav,air_balloon_in['VHor'])
+            dpars['h_b'] = np.max((dpars['h'],10.))
+            dpars['h_u'] = np.max((dpars['h_u'],10.)) #upper limit of mixed layer height
+            dpars['h_l'] = np.max((dpars['h_l'],10.)) #low limit of mixed layer height
+            dpars['h_e'] = np.abs( dpars['h_u'] - dpars['h_l']) # error of mixed-layer height
+            dpars['h'] = np.round(dpars['h_'+air_ap_mode],1)
+        else:
+            dpars['h_u'] =np.nan
+            dpars['h_l'] =np.nan
+            dpars['h_e'] =np.nan
+            dpars['h'] =np.nan
+        
+        
+        
+        if ~np.isnan(dpars['h']):
+            dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]]
+        else:
+            dpars['Ps'] = np.nan
+        
+        if ~np.isnan(dpars['h']):
+        
+            # determine mixed-layer properties (moisture, potential temperature...) from profile
+            
+            # ... and those of the mixed layer
+            is_valid_below_h = is_valid & (air_balloon.z < dpars['h'])
+            valid_indices_below_h =  air_balloon.index[is_valid_below_h].values
+            if len(valid_indices) > 1:
+                if len(valid_indices_below_h) >= 3.:
+                    ml_mean = air_balloon[is_valid_below_h].mean()
+                else:
+                    ml_mean = air_balloon.iloc[valid_indices[0]:valid_indices[1]].mean()
+            elif len(valid_indices) == 1:
+                ml_mean = (air_balloon.iloc[0:1]).mean()
+            else:
+                temp =  pd.DataFrame(air_balloon)
+                temp.iloc[0] = np.nan
+                ml_mean = temp
+                       
+            dpars['theta']= ml_mean.theta
+            dpars['q']    = ml_mean.q
+            dpars['u']    = ml_mean.u
+            dpars['v']    = ml_mean.v 
+        else:
+            dpars['theta'] = np.nan
+            dpars['q'] = np.nan
+            dpars['u'] = np.nan
+            dpars['v'] = np.nan
+        
+        air_ap_head = air_balloon[0:0] #pd.DataFrame(columns = air_balloon.columns)
+        # All other  data points above the mixed-layer fit
+        air_ap_tail = air_balloon[air_balloon.z > dpars['h']]
+
+
+
+
+
+        air_ap_head.z = pd.Series(np.array([2.,dpars['h'],dpars['h']]))
+        jump = air_ap_head.iloc[0] * np.nan
+        
+        
+        if air_ap_tail.shape[0] > 1:
+        
+            # we originally used THTA, but that has another definition than the
+            # variable theta that we need which should be the temperature that
+            # one would have if brought to surface (NOT reference) pressure.
+            for column in ['theta','q','u','v']:
+               
+               # initialize the profile head with the mixed-layer values
+               air_ap_head[column] = ml_mean[column]
+               # calculate jump values at mixed-layer height, which will be
+               # added to the third datapoint of the profile head
+               jump[column] = (air_ap_tail[column].iloc[1]\
+                               -\
+                               air_ap_tail[column].iloc[0])\
+                              /\
+                              (air_ap_tail.z.iloc[1]\
+                               - air_ap_tail.z.iloc[0])\
+                              *\
+                              (dpars['h']- air_ap_tail.z.iloc[0])\
+                              +\
+                              air_ap_tail[column].iloc[0]\
+                              -\
+                              ml_mean[column] 
+               if column == 'theta':
+                  # for potential temperature, we need to set a lower limit to
+                  # avoid the model to crash
+                  jump.theta = np.max((0.1,jump.theta))
+        
+               air_ap_head[column][2] += jump[column]
+        
+        air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 +air_ap_head.v**2)
+
+
+        # filter data so that potential temperature always increases with
+        # height 
+        cols = []
+        for column in air_ap_tail.columns:
+            #if column != 'z':
+                cols.append(column)
+
+        # only select samples monotonically increasing with height
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        for ibottom in range(1,len(air_ap_tail_orig)):
+            if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z +10.:
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],ignore_index=True)
+
+
+
+
+        # make theta increase strong enough to avoid numerical
+        # instability
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        theta_low = air_ap_head['theta'].iloc[2]
+        z_low = air_ap_head['z'].iloc[2]
+        ibottom = 0
+        for itop in range(0,len(air_ap_tail_orig)):
+            theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+            z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+            if (
+                #(z_mean > z_low) and \
+                (z_mean > (z_low+10.)) and \
+                #(theta_mean > (theta_low+0.2) ) and \
+                #(theta_mean > (theta_low+0.2) ) and \
+                 (((theta_mean - theta_low)/(z_mean - z_low)) > 0.0001)):
+
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+                ibottom = itop+1
+                theta_low = air_ap_tail.theta.iloc[-1]
+                z_low =     air_ap_tail.z.iloc[-1]
+            # elif  (itop > len(air_ap_tail_orig)-10):
+            #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        
+        air_ap = \
+            pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+
+
+
+        # # make theta increase strong enough to avoid numerical
+        # # instability
+        # air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        # air_ap_tail = pd.DataFrame()
+        # #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # theta_low = air_ap_head['theta'].iloc[2]
+        # z_low = air_ap_head['z'].iloc[2]
+        # ibottom = 0
+        # for itop in range(0,len(air_ap_tail_orig)):
+        #     theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+        #     z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+        #     if ((theta_mean > (theta_low+0.2) ) and \
+        #          (((theta_mean - theta_low)/(z_mean - z_low)) > 0.001)):
+
+        #         air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+        #         ibottom = itop+1
+        #         theta_low = air_ap_tail.theta.iloc[-1]
+        #         z_low =     air_ap_tail.z.iloc[-1]
+        #     # elif  (itop > len(air_ap_tail_orig)-10):
+        #     #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        # air_ap = \
+        #     pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+        # 
+        # # we copy the pressure at ground level from balloon sounding. The
+        # # pressure at mixed-layer height will be determined internally by class
+        
+        rho        = 1.2                   # density of air [kg m-3]
+        g          = 9.81                  # gravity acceleration [m s-2]
+        
+        air_ap['p'].iloc[0] =dpars['Ps'] 
+        air_ap['p'].iloc[1] =(dpars['Ps'] - rho * g * dpars['h'])
+        air_ap['p'].iloc[2] =(dpars['Ps'] - rho * g * dpars['h'] -0.1)
+        
+        
+        dpars['lat'] = dpars['latitude']
+        # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich)
+        dpars['lon'] = 0.
+        # this is the real longitude that will be used to extract ground data
+        
+        dpars['ldatetime'] = ldate+dt.timedelta(hours=hour)
+        dpars['datetime'] = ldate+dt.timedelta(hours=hour)
+        dpars['doy'] = dpars['datetime'].timetuple().tm_yday
+        
+        dpars['SolarAltitude'] = \
+                                Pysolar.GetAltitude(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        dpars['SolarAzimuth'] =  Pysolar.GetAzimuth(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        
+        
+        dpars['lSunrise'], dpars['lSunset'] \
+        =  Pysolar.util.GetSunriseSunset(dpars['latitude'],
+                                         0.,
+                                         dpars['ldatetime'],0.)
+        
+        # Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually
+        # write local solar time, we need to assign the timezone to UTC (which
+        # is WRONG!!!). Otherwise ruby cannot understand it (it always converts
+        # tolocal computer time :( ). 
+        dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise'])
+        dpars['lSunset'] = pytz.utc.localize(dpars['lSunset'])
+        
+        # This is the nearest datetime when the sun is up (for class)
+        dpars['ldatetime_daylight'] = \
+                                np.min(\
+                                    (np.max(\
+                                        (dpars['ldatetime'],\
+                                         dpars['lSunrise']+dt.timedelta(hours=2))\
+                                     ),\
+                                     dpars['lSunset']\
+                                    )\
+                                )
+        # apply the same time shift for UTC datetime
+        dpars['datetime_daylight'] = dpars['datetime'] \
+                                    +\
+                                    (dpars['ldatetime_daylight']\
+                                     -\
+                                     dpars['ldatetime'])
+        
+        print('ldatetime_daylight',dpars['ldatetime_daylight'])
+        print('ldatetime',dpars['ldatetime'])
+        print('lSunrise',dpars['lSunrise'])
+        dpars['day'] = dpars['ldatetime'].day
+        
+        # We set the starting time to the local sun time, since the model 
+        # thinks we are always at the meridian (lon=0). This way the solar
+        # radiation is calculated correctly.
+        dpars['tstart'] = dpars['ldatetime_daylight'].hour \
+                         + \
+                         dpars['ldatetime_daylight'].minute/60.\
+                         + \
+                         dpars['ldatetime_daylight'].second/3600.
+        
+        print('tstart',dpars['tstart'])
+        dpars['sw_lit'] = False
+        # convert numpy types to native python data types. This provides
+        # cleaner data IO with yaml:
+        for key,value in dpars.items():
+            if type(value).__module__ == 'numpy':
+                dpars[key] = dpars[key].item()
+        
+                decimals = {'p':0,'t':2,'theta':4, 'z':2, 'q':5, 'u':4, 'v':4}
+        # 
+                for column,decimal in decimals.items():
+                    air_balloon[column] = air_balloon[column].round(decimal)
+                    air_ap[column] = air_ap[column].round(decimal)
+        
+        updateglobal = False
+        if c4gli is None:
+            c4gli = class4gl_input()
+            updateglobal = True
+        
+        print('updating...')
+        print(column)
+        c4gli.update(source='bllast',\
+                    # pars=pars,
+                    pars=dpars,\
+                    air_balloon=air_balloon,\
+                    air_ap=air_ap)
+        if updateglobal:
+            c4gli.get_global_input(globaldata)
+
+        # if profile_ini:
+        #     c4gli.runtime = 10 * 3600
+
+        c4gli.dump(file_sounding)
+        
+        # if profile_ini:
+        #     c4gl = class4gl(c4gli)
+        #     c4gl.run()
+        #     c4gl.dump(file_model,\
+        #               include_input=True,\
+        #               timeseries_only=timeseries_only)
+        #     
+        #     # This will cash the observations and model tables per station for
+        #     # the interface
+        # 
+        # if profile_ini:
+        #     profile_ini=False
+        # else:
+        #     profile_ini=True
+        return c4gli
+
+
+path_soundings = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/IOPS_NOON/'
+
+
+file_morning = open(path_soundings+format(current_station.name,'05d')+'_morning.yaml','w') 
+for date,pair  in HOUR_FILES.items(): 
+    print(pair['morning'])
+    humpafn ='/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/BLLAST/MODEM Radiosoundings/'+pair['morning'][1]
+    
+    print(humpafn)
+    balloon_file = open(humpafn,'r',encoding='latin-1')
+
+    c4gli_morning = bllast_parser(balloon_file,file_morning,date,pair['morning'][0])
+    print('c4gli_morning_ldatetime 0',c4gli_morning.pars.ldatetime)
+file_morning.close()
+
+file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_afternoon.yaml','w') 
+for date,pair  in HOUR_FILES.items(): 
+    humpafn ='/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/BLLAST/MODEM Radiosoundings/'+pair['afternoon'][1]
+    balloon_file = open(humpafn,'r',encoding='latin-1')
+
+    c4gli_afternoon = bllast_parser(balloon_file,file_afternoon,date,pair['afternoon'][0])
+    print('c4gli_afternoon_ldatetime 0',c4gli_afternoon.pars.ldatetime)
+file_afternoon.close()
+ 
+
+# file_morning = open(path_soundings+format(current_station.name,'05d')+'_morning.yaml','w') 
+# for date,pair  in HOUR_FILES.items(): 
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/'+pair['morning'][1],
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_morning,hour,c4gli_morning)
+#     print('c4gli_morning_ldatetime 1',c4gli_morning.pars.ldatetime)
+# file_morning.close()
+# 
+# file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_afternoon.yaml','w') 
+# for hour in [18]:
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/humppa_080610_'+format(hour,"02d")+'00.txt'
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_afternoon,hour,c4gli_afternoon)
+# file_afternoon.close()
+
+
+
+# path_model = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/HUMPPA/'
+# 
+# file_model    = open(fnout_model+   format(current_station.name,'05d')+'.yaml','w') 
+
+
+records_morning = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='morning',
+                                           refetch_records=True,
+                                           )
+print('records_morning_ldatetime',records_morning.ldatetime)
+
+records_afternoon = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='afternoon',
+                                           refetch_records=True,
+                                           )
+
+# align afternoon records with noon records, and set same index
+records_afternoon.index = records_afternoon.ldatetime.dt.date
+records_afternoon = records_afternoon.loc[records_morning.ldatetime.dt.date]
+records_afternoon.index = records_morning.index
+path_exp = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/IOPS_NOON/'
+
+os.system('mkdir -p '+path_exp)
+file_morning = open(path_soundings+'/'+format(current_station.name,'05d')+'_morning.yaml')
+file_afternoon = open(path_soundings+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+file_ini = open(path_exp+'/'+format(current_station.name,'05d')+'_ini.yaml','w')
+file_mod = open(path_exp+'/'+format(current_station.name,'05d')+'_mod.yaml','w')
+
+for (STNID,chunk,index),record_morning in records_morning.iterrows():
+    record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+
+    c4gli_morning = get_record_yaml(file_morning, 
+                                    record_morning.index_start, 
+                                    record_morning.index_end,
+                                    mode='ini')
+    #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+    
+    
+    c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                      record_afternoon.index_start, 
+                                      record_afternoon.index_end,
+                                    mode='ini')
+
+    c4gli_morning.update(source='pairs',pars={'runtime' : \
+                        int((c4gli_afternoon.pars.datetime_daylight - 
+                             c4gli_morning.pars.datetime_daylight).total_seconds())})
+
+    
+    c4gli_morning.pars.sw_ac = []
+    c4gli_morning.pars.sw_ap = True
+    c4gli_morning.pars.sw_lit = False
+    c4gli_morning.dump(file_ini)
+    
+    c4gl = class4gl(c4gli_morning)
+    c4gl.run()
+    
+    c4gl.dump(file_mod,\
+              include_input=False,\
+              timeseries_only=timeseries_only)
+file_ini.close()
+file_mod.close()
+file_morning.close()
+file_afternoon.close()
+
+records_ini = get_records(pd.DataFrame([current_station]),\
+                                           path_exp,\
+                                           subset='ini',
+                                           refetch_records=True,
+                                           )
+records_mod = get_records(pd.DataFrame([current_station]),\
+                                           path_exp,\
+                                           subset='mod',
+                                           refetch_records=True,
+                                           )
+
+records_mod.index = records_ini.index
+
+# align afternoon records with initial records, and set same index
+records_afternoon.index = records_afternoon.ldatetime.dt.date
+records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+records_afternoon.index = records_ini.index
+
+
+
+# stations_for_iter = stations(path_exp)
+# for STNID,station in stations_iterator(stations_for_iter):
+#     records_current_station_index = \
+#             (records_ini.index.get_level_values('STNID') == STNID)
+#     file_current_station_mod = STNID
+# 
+#     with \
+#     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+#     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+#     open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+#         for (STNID,index),record_ini in records_iterator(records_ini):
+#             c4gli_ini = get_record_yaml(file_station_ini, 
+#                                         record_ini.index_start, 
+#                                         record_ini.index_end,
+#                                         mode='ini')
+#             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+# 
+#             record_mod = records_mod.loc[(STNID,index)]
+#             c4gl_mod = get_record_yaml(file_station_mod, 
+#                                         record_mod.index_start, 
+#                                         record_mod.index_end,
+#                                         mode='mod')
+#             record_afternoon = records_afternoon.loc[(STNID,index)]
+#             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+#                                         record_afternoon.index_start, 
+#                                         record_afternoon.index_end,
+#                                         mode='ini')
+
+
+# # select the samples of the afternoon list that correspond to the timing of the
+# # morning list
+# records_afternoon = records_afternoon.set_index('ldatetime').loc[records_afternoon.ldatetime)]
+# records_afternoon.index = recods_morning.index
+# 
+# 
+# # create intersectino index
+# index_morning = pd.Index(records_morning.ldatetime.to_date())
+# index_afternoon = pd.Index(records_afternoon.ldatetime.to_date())
+# 
+# for record_morning in records_morning.iterrows():
+#     
+#     c4gl = class4gl(c4gli)
+#     c4gl.run()
+#     c4gl.dump(c4glfile,\
+#               include_input=True,\
+#               timeseries_only=timeseries_only)
+# 
+# # This will cash the observations and model tables per station for
+# # the interface
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=0,\
+#                                    by=2,\
+#                                    subset='ini',
+#                                    refetch_records=True,
+#                                    )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='mod',
+#                                    refetch_records=True,
+#                                    )
+# records_eval = get_records(pd.DataFrame([current_station]),\
+#                                    path_obs,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='eval',
+#                                    refetch_records=True,
+#                                    )
+# 
+# 
+# # mod_scores = pd.DataFrame(index=mod_records.index)
+# # for (STNID,index), current_record_mod in mod_records.iterrows():
+# #     print(STNID,index)
+# #     current_station = STN
+# #     current_record_obs_afternoon = obs_records_afternoon.loc[(STNID,index)]
+# #     current_record_obs = obs_records.loc[(STNID,index)]
+# # 
+# #     record_yaml_mod = get_record_yaml_mod(odirexperiments[keyEXP],\
+# #                                           current_station,\
+# #                                           current_record_mod,\
+# #                                          )
+# # 
+# #     record_yaml_obs = \
+# #             get_record_yaml_obs(odirexperiments[keyEXP],\
+# #                                 current_station,\
+# #                                 current_record_obs,\
+# #                                 suffix='.yaml')
+# # 
+# #     record_yaml_obs_afternoon = \
+# #             get_record_yaml_obs(odir,\
+# #                                 current_station,\
+# #                                 current_record_obs_afternoon,\
+# #                                 suffix='_afternoon.yaml')
+# # 
+# #     hmax = np.max([record_yaml_obs_afternoon.pars.h,\
+# #                    record_yaml_mod.h])
+# #     HEIGHTS = {'h':hmax, '2h':2.*hmax, '3000m':3000.}
+# #     
+# # 
+# #     for height,hvalue in HEIGHTS.items():
+# # 
+# #         lt_obs = (record_yaml_obs_afternoon.air_ap.HAGL < hvalue)
+# #         lt_mod = (record_yaml_mod.air_ap.z < hvalue)
+# #         try:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = \
+# #                 rmse(\
+# #                     record_yaml_obs_afternoon.air_ap.theta[lt_obs],\
+# #                     np.interp(\
+# #                         record_yaml_obs_afternoon.air_ap.HAGL[lt_obs],\
+# #                         record_yaml_mod.air_ap.z[lt_mod],\
+# #                         record_yaml_mod.air_ap.theta[lt_mod]\
+# #                     ))
+# #         except ValueError:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = np.nan
+# #     # # we calculate these things in the interface itself
+# #     # for key in ['q','theta','h']:
+# #     #     mod_records.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_mod.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# #     #     # the actual time of the initial and evaluation sounding can be 
+# #     #     # different, but we consider this as a measurement error for
+# #     #     # the starting and end time of the simulation.
+# #     #     obs_records_afternoon.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_obs_afternoon.pars.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# # mod_scores.to_pickle(odirexperiments[keyEXP]+'/'+format(STNID,'05d')+"_mod_scores.pkl")
+# #         
+# #                 
+# #                 
+# # # for EXP,c4glfile in c4glfiles.items():
+# # #     c4glfile.close()            
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# #     
+# #     # {'Time[min:sec]': None 
+# #     #  'P[hPa]': None, 
+# #     #  'T[C]': None, 
+# #     #  'U[%]': None, 
+# #     #  'Wsp[m/s]': None, 
+# #     #  'Wdir[Grd]': None,
+# #     #  'Lon[°]', 
+# #     #  'Lat[°]', 
+# #     #  'Altitude[m]', 'GeoPot[m']', 'MRI',
+# #     #        'Unnamed: 11', 'RI', 'Unnamed: 13', 'DewPoint[C]', 'Virt. Temp[C]',
+# #     #        'Rs[m/min]D[kg/m3]Azimut[deg]', 'Elevation[deg]', 'Range[m]']
+# #     # }
+# #     # 
+# #     # #pivotrows =
+# #     # #{
+# # 
+# # 
+# # 
diff --git a/class4gl/setup/setup_era.py b/class4gl/setup/setup_era.py
new file mode 100644
index 0000000..6fa6b90
--- /dev/null
+++ b/class4gl/setup/setup_era.py
@@ -0,0 +1,271 @@
+
+
+# -*- coding: utf-8 -*-
+
+import logging
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+import importlib
+spam_loader = importlib.find_loader('Pysolar')
+found = spam_loader is not None
+if found:
+    import Pysolar
+    import Pysolar.util.GetSunriseSunset
+else:
+    import pysolar as Pysolar
+    GetSunriseSunset =  Pysolar.util.get_sunrise_sunset
+
+import argparse
+
+# purpose: create CLASS4GL input data from era-interim
+# example: python $CLASS4GL/setup/setup_era.py --split_by 50 --path_forcing /data/gent/vo/000/gvo00090/D2D/data/SOUNDINGS/ERA_JOAO/ --path_experiments /user/data/gent/gvo000/gvo00090/$USER/data/MY_FIRST_CLASS4GL_EXPERIMENTS/ERA_TEST_JOAO/ --c4gl_path_lib $CLASS4GL  --station_chunk_number 0 --station_id 83779 --force_pars "b=7.12,CGsat=3.670e-06,p=6,a=0.135,C2ref=0.8,C1sat=0.213" --first_YYYYMMDD "20140101"
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--first_YYYYMMDD',default="19810101")
+parser.add_argument('--last_YYYYMMDD',default="20180101")
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--latitude') # run a specific station id
+parser.add_argument('--longitude') # run a specific station id
+parser.add_argument('--error_handling',default='dump_on_success')
+parser.add_argument('--subset_forcing',default='morning') # this tells which yaml subset
+parser.add_argument('--subset_experiments',default='ini') # this tells which yaml subset
+                                                      # to initialize with.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime',default='from_afternoon_profile')
+
+parser.add_argument('--experiments')
+parser.add_argument('--split_by',default=-1)# station soundings are split
+                                            # up in chunks
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--force_pars',default='') # run a specific station id
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+EXP_DEFS  =\
+{
+  'ERA-INTERIM_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'ERA-INTERIM_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'IOPS_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'IOPS_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+
+# iniitialize global data
+# ===============================
+print("Initializing global data")
+# ===============================
+globaldata = data_global()
+globaldata.sources = {**globaldata.sources,**{
+        "ERAINT:t"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/t_6hourly/t_*_6hourly.nc",
+        "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q_*_6hourly.nc",
+        "ERAINT:u"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/u_6hourly/u_*_6hourly.nc",
+        "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v_*_6hourly.nc",
+        }}
+
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+
+
+# ===============================
+print("getting a list of stations")
+# ===============================
+all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+
+
+# # ===============================
+# print("Selecting station by ID")
+# # ===============================
+# stations_iter = stations_iterator(all_stations)
+# STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+# all_stations_select = pd.DataFrame([run_station])
+# print(run_station)
+
+
+# ====================================
+print('defining all_stations_select')
+# ====================================
+
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+if (args.latitude is not None) or (args.longitude is not None):
+    print('custom coordinates not implemented yet, please ask developer.')
+elif args.station_id is not None:
+    print("Selecting station by ID")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+#     print("making a custom station according to the coordinates")
+# 
+#     STNID = 43.23
+else:
+     print("Selecting stations from a row range in the table")
+     all_stations_select = pd.DataFrame(all_stations.table)
+     if args.last_station_row is not None:
+         all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+     if args.first_station_row is not None:
+         all_stations_select = all_station_select.iloc[int(args.first_station):]
+
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+dtfirst = dt.datetime.strptime(args.first_YYYYMMDD,"%Y%m%d").astimezone(dt.timezone.utc)
+dtlast = dt.datetime.strptime(args.last_YYYYMMDD,"%Y%m%d").astimezone(dt.timezone.utc)
+# ===============================
+print("Creating daily timeseries from", dtfirst," to ", dtlast)
+# ===============================
+DTS = [dtfirst + dt.timedelta(days=iday) for iday in \
+       range(int((dtlast + dt.timedelta(days=1) -
+                  dtfirst).total_seconds()/3600./24.))]
+
+if args.split_by != -1:
+    totalchunks = len(all_stations_select)*len(DTS)/int(args.split_by)
+else:
+    totalchunks = len(all_stations_select)
+
+
+if args.global_chunk_number is not None:
+    run_station_chunk = np.mod(int(args.global_chunk_number),len(DTS)/int(args.split_by))
+else:
+    if args.station_chunk_number is not None:
+        run_station_chunk = int(args.station_chunk_number)
+    else:
+        if args.split_by != -1:
+            raise ValueError("Chunks are defined by --split-by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split-by.")
+        run_station_chunk = 0
+        #print("stations that are processed.",list(run_stations.index))
+
+DTS_chunk = DTS[(int(run_station_chunk)*int(args.split_by)):\
+                 (int(run_station_chunk)+1)*int(args.split_by)]
+
+# for the current implementation we only consider one station. Let's upgrade it
+# later for more stations.
+#run_station_chunk = int(args.global_chunk_number)
+
+# ===============================
+print('start looping over chunk')
+# ===============================
+
+os.system('mkdir -p '+args.path_experiments)
+
+
+fn_ini = args.path_experiments+'/'+format(run_station.name,'05d')+'_'+\
+        str(int(run_station_chunk))+'_'+args.subset_experiments+'.yaml'
+file_ini = open(fn_ini,'w');print('Writing to: ',fn_ini)
+
+for iDT,DT in enumerate(DTS_chunk):
+    print(iDT,DT)
+    c4gli = class4gl_input(debug_level=logging.INFO)
+    c4gli.update(source='STNID'+format(STNID,'05d'),\
+                 pars=dict(latitude  = float(run_station.latitude), \
+                           longitude = float(run_station.longitude),\
+                           lat       = float(run_station.latitude), \
+                           # Note the difference between longitude and lon. The
+                           # lon variable should always be zero because we are
+                           # always working in solar time for running CLASS
+                           lon       = 0.,\
+                           STNID     = int(STNID)))
+
+    lSunrise, lSunset = GetSunriseSunset(c4gli.pars.latitude,0.,DT)
+
+    #start simulation at sunrise and stop at one hour before sunset
+    runtime = (lSunset - lSunrise).total_seconds() - 3600.*1.
+    ldatetime = lSunrise
+    datetime = ldatetime - dt.timedelta(hours=c4gli.pars.longitude/360.*24.)
+    datetime_daylight = datetime
+    c4gli.update(source='timeseries',   \
+                 pars=dict(\
+                           lSunrise = lSunrise, \
+                           lSunset = lSunset, \
+                           datetime = datetime, \
+                           ldatetime = ldatetime, \
+                           ldatetime_daylight = ldatetime, \
+                           datetime_daylight = datetime, \
+                           doy = datetime.timetuple().tm_yday,\
+                           runtime = runtime,\
+                           tstart = ldatetime.hour + ldatetime.minute/60. + ldatetime.second/3600.,\
+                          ))
+
+    c4gli.get_global_input(globaldata)
+
+    c4gli.update(source='era-interim',pars={'Ps' : c4gli.pars.sp})
+
+    cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+    Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+    Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+    R = (Rd*(1.-c4gli.air_ac.q) + Rv*c4gli.air_ac.q)
+    rho = c4gli.air_ac.p/R/c4gli.air_ac.t
+    dz = c4gli.air_ac.delpdgrav/rho
+    z = [dz.iloc[-1]/2.]
+    for idz in list(reversed(range(0,len(dz)-1,1))):
+        z.append(z[-1]+ (dz[idz+1]+dz[idz])/2.)
+    z = list(reversed(z))
+
+    theta = c4gli.air_ac.t * \
+               (c4gli.pars.sp/(c4gli.air_ac.p))**(R/cp)
+    thetav   = theta*(1. + 0.61 * c4gli.air_ac.q)
+
+    
+    c4gli.update(source='era-interim',air_ac=pd.DataFrame({'z':list(z),
+                                                           'theta':list(theta),
+                                                           'thetav':list(thetav),
+                                                          }))
+    air_ap_input = c4gli.air_ac[::-1].reset_index().drop('index',axis=1)
+    air_ap_mode = 'b'
+    air_ap_input_source = c4gli.query_source('air_ac:theta')
+
+
+    c4gli.mixed_layer_fit(air_ap=air_ap_input,
+                         source=air_ap_input_source,
+                         mode=air_ap_mode)
+
+    for parkey, parvalue in [par.split('=') for par in args.force_pars.split(',')]:
+        c4gli.update(source='user_specified', pars={parkey: float(parvalue)})
+    if not c4gli.check_source_globaldata():
+        print('Warning: some input sources appear invalid')
+
+    c4gli.dump(file_ini)
+
+file_ini.close()
+all_records_morning = get_records(pd.DataFrame([run_station]),\
+                              args.path_experiments,\
+                              getchunk = int(run_station_chunk),\
+                              subset=args.subset_experiments,
+                              refetch_records=True,
+                              )
+
diff --git a/class4gl/setup/setup_global_afternoon.py b/class4gl/setup/setup_global_afternoon.py
new file mode 100644
index 0000000..e8745dd
--- /dev/null
+++ b/class4gl/setup/setup_global_afternoon.py
@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thursday, March 29, 11:30 AM
+
+@author: Hendrik Wouters
+
+The dry-2-dry global radio sounding experiment.
+
+usage:
+    python setup_global.py 
+    where  is an integer indicating the row index of the station list
+    under args.path_output+'/'+fn_stations (see below)
+
+this scripts should be called from the pbs script setup_global.pbs
+
+
+
+dependencies:
+    - pandas
+    - class4gl
+    - data_soundings
+
+
+"""
+
+""" import libraries """
+import pandas as pd
+import sys
+#import copy as cp
+import numpy as np
+#from sklearn.metrics import mean_squared_error
+import logging
+import datetime as dt
+import os
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_input')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+# parser.add_argument('--first_YYYYMMDD',default="19810101")
+# parser.add_argument('--last_YYYYMMDD',default="20180101")
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--station_id') # run a specific station id
+# parser.add_argument('--error_handling',default='dump_on_success')
+# parser.add_argument('--subset_output',default='morning') # this tells which yaml subset
+
+
+# args.path_output = "/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/GLOBAL/"
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+fn_stations = args.path_input+'/igra-stations.txt'
+
+
+#calculate the root mean square error
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+from class4gl import class4gl_input, data_global,class4gl
+from data_soundings import wyoming
+#from data_global import data_global
+
+# iniitialize global data
+globaldata = data_global()
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+# read the list of stations with valid ground data (list generated with
+# get_valid_stations.py)
+# args.path_input = "/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/"
+
+df_stations = pd.read_fwf(fn_stations,names=['Country code',\
+                                               'ID',\
+                                               'Name',\
+                                               'latitude',\
+                                               'longitude',\
+                                               'height',\
+                                               'unknown',\
+                                               'startyear',\
+                                               'endyear'])
+if args.station_id is not None:
+    df_stations = df_stations[df_stations.ID == int(args.station_id)]
+else:
+    if args.last_station_row is not None:
+        df_stations = df_stations[:(int(args.last_station_row)+1)]
+    if args.first_station_row is not None:
+        df_stations = df_stations[int(args.first_station_row):]
+
+STNlist = list(df_stations.iterrows())
+
+os.system('mkdir -p '+args.path_output)
+for iSTN,STN in STNlist:  
+    one_run = False
+# for iSTN,STN in STNlist[5:]:  
+    
+    fnout = args.path_output+"/"+format(STN['ID'],'05d')+"_afternoon.yaml"
+
+    
+
+    # c4glfiles = dict([(EXP,odirexperiments[EXP]+'/'+format(STN['ID'],'05d')+'.yaml') \
+    #                   for EXP in experiments.keys()])
+        
+    with open(fnout,'w') as fileout:
+        wy_strm = wyoming(PATH=args.path_input, STNM=STN['ID'])
+        wy_strm.set_STNM(int(STN['ID']))
+
+        # we consider all soundings from 1981 onwards
+        wy_strm.find_first(year=1981)
+        #wy_strm.find(dt.datetime(2004,10,19,6))
+        
+        c4gli = class4gl_input(debug_level=logging.INFO)
+        
+        # so we continue as long as we can find a new sounding
+                
+        while wy_strm.current is not None:
+            
+            c4gli.clear()
+            try: 
+                c4gli.get_profile_wyoming(wy_strm)
+                #print(STN['ID'],c4gli.pars.datetime)
+                #c4gli.get_global_input(globaldata)
+
+                print(c4gli.pars.STNID, c4gli.pars.ldatetime)
+
+                logic = dict()
+
+		#still needs to be changed to afternoon
+                logic['morning'] =  (c4gli.pars.ldatetime.hour <= 12.)
+                logic['daylight'] = \
+                    ((c4gli.pars.ldatetime_daylight - 
+                      c4gli.pars.ldatetime).total_seconds()/3600. <= 4.)
+                
+                logic['springsummer'] = (c4gli.pars.theta > 278.)
+                
+                # we take 3000 because previous analysis (ie., HUMPPA) has
+                # focussed towards such altitude
+                le3000 = (c4gli.air_balloon.z <= 3000.)
+                logic['10measurements'] = (np.sum(le3000) >= 5) 
+
+                #leh = (c4gli.air_balloon.z <= c4gli.pars.h)
+
+                #logic['mlerrlow'] = (\
+                #        (len(np.where(leh)[0]) > 0) and \
+                        # in cases where humidity is not defined, the mixed-layer
+                        # values get corr
+                #        (not np.isnan(c4gli.pars.theta)) and \
+                #        (rmse(c4gli.air_balloon.theta[leh] , \
+                #              c4gli.pars.theta,filternan_actual=True) < 1.)\
+                #              )
+    
+
+                #logic['mlherrlow'] = (c4gli.pars.h_e <= 150.)
+                    
+                print('logic:', logic)
+                # the result
+                morning_ok = np.mean(list(logic.values()))
+                print(morning_ok,c4gli.pars.ldatetime)
+
+            except:
+                morning_ok =False
+                print('obtain morning not good')
+
+            # the next sounding will be used either for an afternoon sounding
+            # or for the morning sounding of the next day.
+            
+            # If the morning is ok, then we try to find a decent afternoon
+            # sounding
+                                
+            print(morning_ok)
+            if morning_ok == 1.:
+                c4gli.get_global_input(globaldata)
+                print('VERY CLOSE...')
+                if c4gli.check_source_globaldata() and \
+	            (c4gli.check_source(source='wyoming',\
+		                                   check_only_sections='pars')):
+                    c4gli.dump(fileout)
+                    one_run=True		     
+                    print('HIT!!!')
+            wy_strm.find_next()
+                
+    if one_run:
+        STN.name = STN['ID']
+
+        all_records_afternoon = get_records(pd.DataFrame([STN]),\
+                                      args.path_output,\
+                                      subset='afternoon',
+                                      refetch_records=True,
+                                      )
+    else:
+        os.system('rm '+fnout)
+        
+
+    # for c4glfile in c4glfiles:
+    #     c4glfile.close()            
+
diff --git a/class4gl/setup/setup_goamazon.py b/class4gl/setup/setup_goamazon.py
new file mode 100644
index 0000000..18f5612
--- /dev/null
+++ b/class4gl/setup/setup_goamazon.py
@@ -0,0 +1,834 @@
+# -*- coding: utf-8 -*-
+
+import xarray as xr
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import Pysolar
+import sys
+import pytz
+import glob
+sys.path.insert(0,'/user/home/gent/vsc422/vsc42247/software/class4gl/class4gl/')
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+
+globaldata = data_global()
+globaldata.load_datasets(recalc=0)
+
+Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+epsilon = Rd/Rv # or mv/md
+
+path_soundings_in = '/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/GOAMAZON/radio_all/'
+
+def replace_iter(iterable, search, replace):
+    for value in iterable:
+        value.replace(search, replace)
+        yield value
+
+from class4gl import blh,class4gl_input
+
+# definition of the humpa station
+current_station = pd.Series({ "latitude"  : -3.21,
+                  "longitude" : -60.6,
+                  "name" : "the GOAMAZON experiment"
+                })
+current_station.name = 90002
+
+# we define the columns ourselves because it is a mess in the file itself.
+columns =\
+['Time[min:sec]',
+ 'P[hPa]',
+ 'T[C]',
+ 'U[%]',
+ 'Wsp[m/s]',
+ 'Wdir[Grd]',
+ 'Lon[°]',
+ 'Lat[°]',
+ 'Altitude[m]',
+ 'GeoPot[m]',
+ 'MRI',
+ 'RI',    
+ 'DewPoint[C]',
+ 'Virt. Temp[C]',
+ 'Rs[m/min]',
+ 'D[kg/m3]',
+ 'Azimut[°]',
+ 'Elevation[°]',
+ 'Range[m]',
+]
+
+DTSTART = dt.datetime(2014,1,1,0,0,0,0,pytz.UTC)
+DTEND = dt.datetime(2015,5,16,0,0,0,0,pytz.UTC)
+
+
+DTS = [DTSTART+dt.timedelta(days=day) for day in range(0, int((DTEND-DTSTART).total_seconds()/3600./24.))]
+HOUR_FILES = {}
+for iDT, DT in enumerate(DTS):
+    morning_file = None
+    possible_files_morning =\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.11??00.*cdf')+\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.10??00.*cdf')+\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.09??00.*cdf')+\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.08??00.*cdf')+\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.12??00.*cdf')+\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.13??00.*cdf')+\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.14??00.*cdf')+\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.15??00.*cdf')
+    # glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.07??00.*cdf')+\
+    # glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.06??00.*cdf')+\
+    # glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.05??00.*cdf')+\
+    if len(possible_files_morning)>0:
+        ix = 0
+        while ((ix < (len(possible_files_morning))) and (morning_file is None)):
+            # print (xr.open_dataset(possible_files_morning[ix]).pres.shape)
+            # print(xr.open_dataset(possible_files_morning[ix]).pres.shape[0] >= 10)
+            if (xr.open_dataset(possible_files_morning[ix]).pres.shape[0] >= 500) and\
+                (possible_files_morning[ix].split('/')[-1] != 'maosondewnpnM1.b1.20150103.115200.custom.cdf') and \
+                (possible_files_morning[ix].split('/')[-1] != 'maosondewnpnM1.b1.20150103.115200.custom.cdf'):
+                morning_file = possible_files_morning[ix]
+            ix +=1
+
+    afternoon_file = None
+    possible_files_afternoon =\
+        glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.20??00.*cdf')+\
+        glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.19??00.*cdf')+\
+        glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.18??00.*cdf')+\
+        glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.17??00.*cdf')+\
+        glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.16??00.*cdf')
+    if len(possible_files_afternoon)>0:
+        ix = 0
+        while ((ix < (len(possible_files_afternoon))) and (afternoon_file is None)):
+            # print (xr.open_dataset(possible_files_afternoon[ix]).pres.shape)
+            # print(xr.open_dataset(possible_files_afternoon[ix]).pres.shape[0] >= 10)
+            if (xr.open_dataset(possible_files_afternoon[ix]).pres.shape[0] >= 300) and \
+               (possible_files_afternoon[ix].split('/')[-1] != '/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/GOAMAZON/Radiosounding_ARM/maosondewnpnM1.b1.20140823.143600.cdf'):
+                afternoon_file = possible_files_afternoon[ix]
+            ix +=1
+
+    if (morning_file is not None) and (afternoon_file is not None):
+        HOUR_FILES[DT] = {'morning':  [int(morning_file.split('/')[-1].split('.')[3])/10000.,morning_file],
+                          'afternoon':[ int(afternoon_file.split('/')[-1].split('.')[3])/10000. ,afternoon_file]}
+        print(HOUR_FILES[DT])
+
+# HOUR_FILES = \
+# {
+#     dt.datetime(2015,5,7,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150507.052900.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150507.172700.custom.cdf']},
+#     dt.datetime(2015,3,13,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150313.052700.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150313.173000.custom.cdf']},
+#     dt.datetime(2015,3,12,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150312.052800.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150312.173400.custom.cdf']},
+#     dt.datetime(2015,3,12,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150312.052800.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150312.173400.custom.cdf']},
+#     dt.datetime(2015,3,12,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150312.052800.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150312.173400.custom.cdf']},
+# }
+
+
+
+
+#only include the following timeseries in the model output
+timeseries_only = \
+['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+ 'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+ 'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+ 'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+ 'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+def esat(T):
+    return 0.611e3 * np.exp(17.2694 * (T - 273.16) / (T - 35.86))
+def efrom_rh100_T(rh100,T):
+    return esat(T)*rh100/100.
+def qfrom_e_p(e,p):
+    return epsilon * e/(p - (1.-epsilon)*e)
+
+
+
+def humppa_parser(balloon_file,file_sounding,ldate,lhour,c4gli=None):
+        print(balloon_file)
+        
+        xrin = balloon_file
+        air_balloon = pd.DataFrame()
+
+        air_balloon['t'] = xrin.tdry.values+273.15
+        air_balloon['p'] = xrin.pres.values*100.
+        
+        air_balloon['u'] = xrin.u_wind.values
+        air_balloon['v'] = xrin.v_wind.values
+        air_balloon['WSPD'] = xrin['wspd'].values
+        
+        print(xrin.rh.values.shape)
+        air_balloon['q'] = qfrom_e_p(efrom_rh100_T(xrin.rh.values,air_balloon['t'].values),air_balloon.p.values)
+        
+
+        #balloon_conv = replace_iter(balloon_file,"°","deg")
+        #readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()]
+        #air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15)
+        # air_balloon_in = pd.read_fwf(balloon_file,
+        #                              widths=[14]*19,
+        #                              skiprows=9,
+        #                              skipfooter=15,
+        #                              decimal=',',
+        #                              header=None,
+        #                              names = columns,
+        #                              na_values='-----')
+    
+
+        
+        rowmatches = {
+            'R' :    lambda x: (Rd*(1.-x.q) + Rv*x.q),
+            'theta': lambda x: (x['t']) * (x['p'][0]/x['p'])**(x['R']/cp),
+            'thetav': lambda x: x.theta  + 0.61 * x.theta * x.q,
+            'rho': lambda x: x.p /x.t / x.R ,
+        }
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon)
+        
+        print('alt in xrin?:','alt' in xrin)
+        if 'alt' in xrin:
+            air_balloon['z'] = xrin.alt.values
+        else:
+            g          = 9.81                  # gravity acceleration [m s-2]
+            air_balloon['z'] = 0.
+            for irow,row in air_balloon.iloc[1:].iterrows():
+                air_balloon['z'].iloc[irow] = air_balloon['z'].iloc[irow-1] - \
+                        2./(air_balloon['rho'].iloc[irow-1]+air_balloon['rho'].iloc[irow])/g * \
+                        (air_balloon['p'].iloc[irow] - air_balloon['p'].iloc[irow-1])
+                    
+             
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon)
+        
+        dpars = {}
+        dpars['longitude']  = current_station['longitude']
+        dpars['latitude']  = current_station['latitude'] 
+        
+        dpars['STNID'] = current_station.name
+        
+
+        # # there are issues with the lower measurements in the HUMPPA campaign,
+        # # for which a steady decrease of potential temperature is found, which
+        # # is unrealistic.  Here I filter them away
+        # ifirst = 0
+        # while  (air_balloon.theta.iloc[ifirst+1] < air_balloon.theta.iloc[ifirst]):
+        #     ifirst = ifirst+1
+        # print ('ifirst:',ifirst)
+        # air_balloon = air_balloon.iloc[ifirst:].reset_index().drop(['index'],axis=1)
+        air_balloon = air_balloon.iloc[:].reset_index().drop(['index'],axis=1)
+        
+        # if air_balloon.z.max() > 100000.:
+        #     air_balloon.z = air_balloon.z/10.
+
+        is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)
+        valid_indices = air_balloon.index[is_valid].values
+        
+        air_ap_mode='b'
+        
+        i = 1
+        while (air_balloon.thetav.iloc[valid_indices[0]] - \
+               air_balloon.thetav.iloc[valid_indices[i]] ) > 0.5:
+            #diff = (air_balloon.theta.iloc[valid_indices[i]] -air_balloon.theta.iloc[valid_indices[i+1]])- 0.5
+            air_balloon.thetav.iloc[valid_indices[0:i]] = \
+                air_balloon.thetav.iloc[valid_indices[i]] + 0.5 
+            
+            i +=1
+
+
+        # while ((len(valid_indices) > 10) and
+        #        ((air_balloon.theta.iloc[valid_indices[0]] -
+        #          air_balloon.theta.iloc[valid_indices[1]]) > 0.5)):
+        #     valid_indices = valid_indices[1:]
+
+        #theta_vs_first_inconsistent = True
+        # while theta_vs_first_inconsistent:
+        
+        if len(valid_indices) > 0:
+            air_balloon_temp = air_balloon.iloc[valid_indices]
+            print(air_balloon_temp)
+            print(air_balloon_temp.z.shape,air_balloon_temp.thetav.shape,)
+            dpars['h'],dpars['h_u'],dpars['h_l'] =\
+                blh(air_balloon_temp.z.values,air_balloon_temp.thetav.values,air_balloon_temp.WSPD.values)
+            dpars['h_b'] = np.max((dpars['h'],10.))
+            dpars['h_u'] = np.max((dpars['h_u'],10.)) #upper limit of mixed layer height
+            dpars['h_l'] = np.max((dpars['h_l'],10.)) #low limit of mixed layer height
+            dpars['h_e'] = np.abs( dpars['h_u'] - dpars['h_l']) # error of mixed-layer height
+            dpars['h'] = np.round(dpars['h_'+air_ap_mode],1)
+        else:
+            dpars['h_u'] =np.nan
+            dpars['h_l'] =np.nan
+            dpars['h_e'] =np.nan
+            dpars['h'] =np.nan
+        
+        
+        
+        if ~np.isnan(dpars['h']):
+            dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]]
+        else:
+            dpars['Ps'] = np.nan
+        
+        if ~np.isnan(dpars['h']):
+        
+            # determine mixed-layer properties (moisture, potential temperature...) from profile
+            
+            # ... and those of the mixed layer
+            is_valid_below_h = (air_balloon.iloc[valid_indices].z < dpars['h'])
+            valid_indices_below_h =  air_balloon.iloc[valid_indices].index[is_valid_below_h].values
+            if len(valid_indices) > 1:
+                if len(valid_indices_below_h) >= 3.:
+                    ml_mean = air_balloon.iloc[valid_indices][is_valid_below_h].mean()
+                else:
+                    ml_mean = air_balloon.iloc[valid_indices[0]:valid_indices[1]].mean()
+            elif len(valid_indices) == 1:
+                ml_mean = (air_balloon.iloc[0:1]).mean()
+            else:
+                temp =  pd.DataFrame(air_balloon)
+                temp.iloc[0] = np.nan
+                ml_mean = temp
+                       
+
+
+            dpars['theta']= ml_mean.theta
+            dpars['q']    = ml_mean.q
+            dpars['u']    = ml_mean.u
+            dpars['v']    = ml_mean.v 
+            # theta_vs_first_inconsistent = \
+            #     ((air_balloon.theta.iloc[valid_indices[0]] - air_balloon.theta.iloc[valid_indices[1]]) > 0.2)
+            # theta_vs_first_inconsistent = \
+            #     ((air_balloon.theta.iloc[valid_indices[0]] - dpars['theta']) > 0.1)
+            # if theta_vs_first_inconsistent:
+            #     valid_indices = valid_indices[1:]
+            #     print("warning! too large difference between near surface value and abl value of theta. I'm taking the next one as near surface vlue")
+        else:
+            dpars['theta'] = np.nan
+            dpars['q'] = np.nan
+            dpars['u'] = np.nan
+            dpars['v'] = np.nan
+            # theta_bl_inconsistent = False
+
+        
+        air_ap_head = air_balloon[0:0] #pd.DataFrame(columns = air_balloon.columns)
+        # All other  data points above the mixed-layer fit
+        air_ap_tail = air_balloon[air_balloon.z > dpars['h']]
+
+
+
+        air_ap_head.z = pd.Series(np.array([2.,dpars['h'],dpars['h']]))
+        jump = air_ap_head.iloc[0] * np.nan
+        
+        if air_ap_tail.shape[0] > 1:
+        
+            # we originally used THTA, but that has another definition than the
+            # variable theta that we need which should be the temperature that
+            # one would have if brought to surface (NOT reference) pressure.
+            for column in ['theta','q','u','v']:
+               
+               # initialize the profile head with the mixed-layer values
+               air_ap_head[column] = ml_mean[column]
+               # calculate jump values at mixed-layer height, which will be
+               # added to the third datapoint of the profile head
+               jump[column] = (air_ap_tail[column].iloc[1]\
+                               -\
+                               air_ap_tail[column].iloc[0])\
+                              /\
+                              (air_ap_tail.z.iloc[1]\
+                               - air_ap_tail.z.iloc[0])\
+                              *\
+                              (dpars['h']- air_ap_tail.z.iloc[0])\
+                              +\
+                              air_ap_tail[column].iloc[0]\
+                              -\
+                              ml_mean[column] 
+               if column == 'theta':
+                  # for potential temperature, we need to set a lower limit to
+                  # avoid the model to crash
+                  jump.theta = np.max((0.1,jump.theta))
+        
+               air_ap_head[column][2] += jump[column]
+        
+        air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 +air_ap_head.v**2)
+
+
+
+        # only select samples monotonically increasing with height
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        print(air_ap_tail_orig)
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        for ibottom in range(1,len(air_ap_tail_orig)):
+            if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z +10.:
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],ignore_index=True)
+
+
+        # make theta increase strong enough to avoid numerical
+        # instability
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        # air_ap_tail = pd.DataFrame()
+        # #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # theta_low = air_ap_head['theta'].iloc[2]
+        # z_low = air_ap_head['z'].iloc[2]
+        # ibottom = 0
+        # for itop in range(0,len(air_ap_tail_orig)):
+        #     theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+        #     z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+        #     if (
+        #         #(z_mean > z_low) and \
+        #         (z_mean > (z_low+10.)) and \
+        #         #(theta_mean > (theta_low+0.2) ) and \
+        #         #(theta_mean > (theta_low+0.2) ) and \
+        #          (((theta_mean - theta_low)/(z_mean - z_low)) > 0.00001)):
+
+        #         air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+        #         ibottom = itop+1
+        #         theta_low = air_ap_tail.theta.iloc[-1]
+        #         z_low =     air_ap_tail.z.iloc[-1]
+        #     # elif  (itop > len(air_ap_tail_orig)-10):
+        #     #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        # 
+        air_ap = \
+            pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+        
+        # we copy the pressure at ground level from balloon sounding. The
+        # pressure at mixed-layer height will be determined internally by class
+        
+        rho        = 1.2                   # density of air [kg m-3]
+        g          = 9.81                  # gravity acceleration [m s-2]
+        
+        air_ap['p'].iloc[0] =dpars['Ps'] 
+        air_ap['p'].iloc[1] =(dpars['Ps'] - rho * g * dpars['h'])
+        air_ap['p'].iloc[2] =(dpars['Ps'] - rho * g * dpars['h'] -0.1)
+        
+        
+        dpars['lat'] = dpars['latitude']
+        # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich)
+        dpars['lon'] = 0.
+        # this is the real longitude that will be used to extract ground data
+        
+        # dpars['ldatetime'] = ldate+dt.timedelta(hours=lhour)
+        # dpars['datetime'] =  dpars['ldatetime'] + dt.timedelta(hours=+4)
+
+
+        dpars['datetime'] =  ldate+dt.timedelta(hours=lhour)
+        dpars['ldatetime'] =  dpars['datetime'] + dt.timedelta(hours=dpars['longitude']/360.*24.)
+
+
+        dpars['doy'] = dpars['datetime'].timetuple().tm_yday
+        
+        dpars['SolarAltitude'] = \
+                                Pysolar.GetAltitude(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        dpars['SolarAzimuth'] =  Pysolar.GetAzimuth(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        
+        
+        dpars['lSunrise'], dpars['lSunset'] \
+        =  Pysolar.util.GetSunriseSunset(dpars['latitude'],
+                                         0.,
+                                         dpars['ldatetime'],0.)
+        
+        # Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually write local solar time, we need to assign the timezone to UTC (which is WRONG!!!). Otherwise ruby cannot understand it (it always converts tolocal computer time :( ). 
+        dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise'])
+        dpars['lSunset'] = pytz.utc.localize(dpars['lSunset'])
+        
+        # This is the nearest datetime when the sun is up (for class)
+        dpars['ldatetime_daylight'] = \
+                                np.min(\
+                                    (np.max(\
+                                        (dpars['ldatetime'],\
+                                         dpars['lSunrise'])\
+                                     ),\
+                                     dpars['lSunset']\
+                                    )\
+                                )
+        # apply the same time shift for UTC datetime
+        dpars['datetime_daylight'] = dpars['datetime'] \
+                                    +\
+                                    (dpars['ldatetime_daylight']\
+                                     -\
+                                     dpars['ldatetime'])
+        
+        
+        # We set the starting time to the local sun time, since the model 
+        # thinks we are always at the meridian (lon=0). This way the solar
+        # radiation is calculated correctly.
+        dpars['tstart'] = dpars['ldatetime_daylight'].hour \
+                         + \
+                         dpars['ldatetime_daylight'].minute/60.\
+                         + \
+                         dpars['ldatetime_daylight'].second/3600.
+        
+        dpars['sw_lit'] = False
+        # convert numpy types to native python data types. This provides
+        # cleaner data IO with yaml:
+        for key,value in dpars.items():
+            if type(value).__module__ == 'numpy':
+                dpars[key] = dpars[key].item()
+        
+                decimals = {'p':0,'t':2,'theta':4, 'z':2, 'q':5, 'u':4, 'v':4}
+        # 
+                for column,decimal in decimals.items():
+                    air_balloon[column] = air_balloon[column].round(decimal)
+                    air_ap[column] = air_ap[column].round(decimal)
+        
+
+
+        dpars['gammatheta_lower_limit'] = 0.0001
+        updateglobal = False
+        if c4gli is None:
+            c4gli = class4gl_input()
+            updateglobal = True
+        
+        print('updating...')
+        print(column)
+        c4gli.update(source='goamazon',\
+                    # pars=pars,
+                    pars=dpars,\
+                    air_balloon=air_balloon,\
+                    air_ap=air_ap)
+        if updateglobal:
+            c4gli.get_global_input(globaldata)
+
+        # if profile_ini:
+        #     c4gli.runtime = 10 * 3600
+
+        # if not ((dpars['ldatetime'].hour <=12) or\
+        #    ((dpars['lSunset'].hour - dpars['ldatetime'].hour) >= (2.))):
+        #     c4gli = None
+        
+        # if profile_ini:
+        #     c4gl = class4gl(c4gli)
+        #     c4gl.run()
+        #     c4gl.dump(file_model,\
+        #               include_input=True,\
+        #               timeseries_only=timeseries_only)
+        #     
+        #     # This will cash the observations and model tables per station for
+        #     # the interface
+        # 
+        # if profile_ini:
+        #     profile_ini=False
+        # else:
+        #     profile_ini=True
+        return c4gli
+
+
+# path_soundings = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/IOPS/'
+path_soundings = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/GOAMAZON6/'
+
+if os.path.isdir(path_soundings):
+    print("Warning, I'm removing "+path_soundings+" in 10 seconds. Press ctrl-c to cancel")
+    os.system('rm '+path_soundings)
+file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_end.yaml','w') 
+file_morning = open(path_soundings+format(current_station.name,'05d')+'_ini.yaml','w') 
+ipair = 0
+for date,pair  in HOUR_FILES.items(): 
+    
+    print(date,ipair,pair)
+    humpafn = pair['afternoon'][1]
+    balloon_file_afternoon = xr.open_dataset(humpafn)
+    humpafn = pair['morning'][1]
+    balloon_file_morning = xr.open_dataset(humpafn)
+    if (\
+        (balloon_file_morning.pres.shape[0] > 10) and \
+        (balloon_file_afternoon.pres.shape[0] > 10)\
+        ):
+        print('filename',pair['afternoon'][1],date,pair['afternoon'][0])
+        c4gli_afternoon = humppa_parser(balloon_file_afternoon,file_afternoon,date,pair['afternoon'][0])
+        ipair += 1
+        
+
+        print('filename',pair['morning'][1],date,pair['afternoon'][0])
+        c4gli_morning = humppa_parser(balloon_file_morning,file_morning,date,pair['morning'][0])
+        if (c4gli_morning is not None) and (c4gli_afternoon is not None):
+            print('c4gli_morning_ldatetime 0',c4gli_morning.pars.ldatetime)
+            print('c4gli_afternoon_ldatetime 0',c4gli_afternoon.pars.ldatetime)
+            c4gli_morning.dump(file_morning)
+            c4gli_afternoon.dump(file_afternoon)
+
+
+print(ipair)
+file_afternoon.close()
+file_morning.close()
+
+ 
+
+# file_morning = open(path_soundings+format(current_station.name,'05d')+'_morning.yaml','w') 
+# for date,pair  in HOUR_FILES.items(): 
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/GOAMAZON/Radiosounding_ARM/'+pair['morning'][1],
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_morning,hour,c4gli_morning)
+#     print('c4gli_morning_ldatetime 1',c4gli_morning.pars.ldatetime)
+# file_morning.close()
+# 
+# file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_afternoon.yaml','w') 
+# for hour in [18]:
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/GOAMAZON/Radiosounding_ARM//humppa_080610_'+format(hour,"02d")+'00.txt'
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_afternoon,hour,c4gli_afternoon)
+# file_afternoon.close()
+
+
+
+# path_model = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/HUMPPA/'
+# 
+# file_model    = open(fnout_model+   format(current_station.name,'05d')+'.yaml','w') 
+
+records_morning = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='ini',\
+                                           refetch_records=True,\
+                                           )
+print('records_morning_ldatetime',records_morning.ldatetime)
+
+records_afternoon = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='end',\
+                                           refetch_records=True,\
+                                           )
+
+# align afternoon records with noon records, and set same index
+records_afternoon.index = records_afternoon.ldatetime.dt.date
+records_afternoon = records_afternoon.loc[records_morning.ldatetime.dt.date]
+records_afternoon.index = records_morning.index
+path_exp = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/IOPS/'
+
+# os.system('mkdir -p '+path_exp)
+# file_morning = open(path_soundings+'/'+format(current_station.name,'05d')+'_ini.yaml')
+# file_afternoon = open(path_soundings+'/'+format(current_station.name,'05d')+'_end.yaml')
+# file_ini = open(path_exp+'/'+format(current_station.name,'05d')+'_ini.yaml','w')
+# file_mod = open(path_exp+'/'+format(current_station.name,'05d')+'_mod.yaml','w')
+# 
+# for (STNID,chunk,index),record_morning in records_morning.iterrows():
+#     record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+# 
+#     c4gli_morning = get_record_yaml(file_morning, 
+#                                     record_morning.index_start, 
+#                                     record_morning.index_end,
+#                                     mode='ini')
+#     #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+#     
+#     
+#     c4gli_afternoon = get_record_yaml(file_afternoon, 
+#                                       record_afternoon.index_start, 
+#                                       record_afternoon.index_end,
+#                                     mode='ini')
+# 
+#     c4gli_morning.update(source='pairs',pars={'runtime' : \
+#                         int((c4gli_afternoon.pars.datetime_daylight - 
+#                              c4gli_morning.pars.datetime_daylight).total_seconds())})
+#     c4gli_morning.update(source='manual',
+#                          pars={'sw_ac' : [],'sw_ap': True,'sw_lit': False})
+#     c4gli_morning.dump(file_ini)
+#     
+#     c4gl = class4gl(c4gli_morning)
+#     c4gl.run()
+#     
+#     c4gl.dump(file_mod,\
+#               include_input=False,\
+#               timeseries_only=timeseries_only)
+# file_ini.close()
+# file_mod.close()
+# file_morning.close()
+# file_afternoon.close()
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                            path_exp,\
+#                                            subset='ini',
+#                                            refetch_records=True,
+#                                            )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                            path_exp,\
+#                                            subset='mod',
+#                                            refetch_records=True,
+#                                            )
+# 
+# records_mod.index = records_ini.index
+# 
+# # align afternoon records with initial records, and set same index
+# records_afternoon.index = records_afternoon.ldatetime.dt.date
+# records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+# records_afternoon.index = records_ini.index
+
+
+"""
+stations_for_iter = stations(path_exp)
+for STNID,station in stations_iterator(stations_for_iter):
+    records_current_station_index = \
+            (records_ini.index.get_level_values('STNID') == STNID)
+    file_current_station_mod = STNID
+
+    with \
+    open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+    open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+        for (STNID,index),record_ini in records_iterator(records_ini):
+            c4gli_ini = get_record_yaml(file_station_ini, 
+                                        record_ini.index_start, 
+                                        record_ini.index_end,
+                                        mode='ini')
+            #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+
+            record_mod = records_mod.loc[(STNID,index)]
+            c4gl_mod = get_record_yaml(file_station_mod, 
+                                        record_mod.index_start, 
+                                        record_mod.index_end,
+                                        mode='mod')
+            record_afternoon = records_afternoon.loc[(STNID,index)]
+            c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+                                        record_afternoon.index_start, 
+                                        record_afternoon.index_end,
+                                        mode='ini')
+"""
+
+
+# # select the samples of the afternoon list that correspond to the timing of the
+# # morning list
+# records_afternoon = records_afternoon.set_index('ldatetime').loc[records_afternoon.ldatetime)]
+# records_afternoon.index = recods_morning.index
+# 
+# 
+# # create intersectino index
+# index_morning = pd.Index(records_morning.ldatetime.to_date())
+# index_afternoon = pd.Index(records_afternoon.ldatetime.to_date())
+# 
+# for record_morning in records_morning.iterrows():
+#     
+#     c4gl = class4gl(c4gli)
+#     c4gl.run()
+#     c4gl.dump(c4glfile,\
+#               include_input=True,\
+#               timeseries_only=timeseries_only)
+# 
+# # This will cash the observations and model tables per station for
+# # the interface
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=0,\
+#                                    by=2,\
+#                                    subset='ini',
+#                                    refetch_records=True,
+#                                    )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='mod',
+#                                    refetch_records=True,
+#                                    )
+# records_eval = get_records(pd.DataFrame([current_station]),\
+#                                    path_obs,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='eval',
+#                                    refetch_records=True,
+#                                    )
+# 
+# 
+# # mod_scores = pd.DataFrame(index=mod_records.index)
+# # for (STNID,index), current_record_mod in mod_records.iterrows():
+# #     print(STNID,index)
+# #     current_station = STN
+# #     current_record_obs_afternoon = obs_records_afternoon.loc[(STNID,index)]
+# #     current_record_obs = obs_records.loc[(STNID,index)]
+# # 
+# #     record_yaml_mod = get_record_yaml_mod(odirexperiments[keyEXP],\
+# #                                           current_station,\
+# #                                           current_record_mod,\
+# #                                          )
+# # 
+# #     record_yaml_obs = \
+# #             get_record_yaml_obs(odirexperiments[keyEXP],\
+# #                                 current_station,\
+# #                                 current_record_obs,\
+# #                                 suffix='.yaml')
+# # 
+# #     record_yaml_obs_afternoon = \
+# #             get_record_yaml_obs(odir,\
+# #                                 current_station,\
+# #                                 current_record_obs_afternoon,\
+# #                                 suffix='_afternoon.yaml')
+# # 
+# #     hmax = np.max([record_yaml_obs_afternoon.pars.h,\
+# #                    record_yaml_mod.h])
+# #     HEIGHTS = {'h':hmax, '2h':2.*hmax, '3000m':3000.}
+# #     
+# # 
+# #     for height,hvalue in HEIGHTS.items():
+# # 
+# #         lt_obs = (record_yaml_obs_afternoon.air_ap.HAGL < hvalue)
+# #         lt_mod = (record_yaml_mod.air_ap.z < hvalue)
+# #         try:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = \
+# #                 rmse(\
+# #                     record_yaml_obs_afternoon.air_ap.theta[lt_obs],\
+# #                     np.interp(\
+# #                         record_yaml_obs_afternoon.air_ap.HAGL[lt_obs],\
+# #                         record_yaml_mod.air_ap.z[lt_mod],\
+# #                         record_yaml_mod.air_ap.theta[lt_mod]\
+# #                     ))
+# #         except ValueError:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = np.nan
+# #     # # we calculate these things in the interface itself
+# #     # for key in ['q','theta','h']:
+# #     #     mod_records.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_mod.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# #     #     # the actual time of the initial and evaluation sounding can be 
+# #     #     # different, but we consider this as a measurement error for
+# #     #     # the starting and end time of the simulation.
+# #     #     obs_records_afternoon.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_obs_afternoon.pars.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# # mod_scores.to_pickle(odirexperiments[keyEXP]+'/'+format(STNID,'05d')+"_mod_scores.pkl")
+# #         
+# #                 
+# #                 
+# # # for EXP,c4glfile in c4glfiles.items():
+# # #     c4glfile.close()            
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# #     
+# #     # {'Time[min:sec]': None 
+# #     #  'P[hPa]': None, 
+# #     #  'T[C]': None, 
+# #     #  'U[%]': None, 
+# #     #  'Wsp[m/s]': None, 
+# #     #  'Wdir[Grd]': None,
+# #     #  'Lon[°]', 
+# #     #  'Lat[°]', 
+# #     #  'Altitude[m]', 'GeoPot[m']', 'MRI',
+# #     #        'Unnamed: 11', 'RI', 'Unnamed: 13', 'DewPoint[C]', 'Virt. Temp[C]',
+# #     #        'Rs[m/min]D[kg/m3]Azimut[deg]', 'Elevation[deg]', 'Range[m]']
+# #     # }
+# #     # 
+# #     # #pivotrows =
+# #     # #{
+# # 
+# # 
+# # 
diff --git a/class4gl/setup/setup_goamazon_noon.py b/class4gl/setup/setup_goamazon_noon.py
new file mode 100644
index 0000000..efd9333
--- /dev/null
+++ b/class4gl/setup/setup_goamazon_noon.py
@@ -0,0 +1,753 @@
+# -*- coding: utf-8 -*-
+
+import xarray as xr
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import Pysolar
+import sys
+import pytz
+import glob
+sys.path.insert(0,'/user/home/gent/vsc422/vsc42247/software/class4gl/class4gl/')
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+
+globaldata = data_global()
+globaldata.load_datasets(recalc=0)
+
+Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+epsilon = Rd/Rv # or mv/md
+
+path_soundings_in = '/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/GOAMAZON/Radiosounding_ARM/'
+
+def replace_iter(iterable, search, replace):
+    for value in iterable:
+        value.replace(search, replace)
+        yield value
+
+from class4gl import blh,class4gl_input
+
+# definition of the humpa station
+current_station = pd.Series({ "latitude"  : -3.21,
+                  "longitude" : -60.6,
+                  "name" : "the GOAMAZON experiment"
+                })
+current_station.name = 90002
+
+# we define the columns ourselves because it is a mess in the file itself.
+columns =\
+['Time[min:sec]',
+ 'P[hPa]',
+ 'T[C]',
+ 'U[%]',
+ 'Wsp[m/s]',
+ 'Wdir[Grd]',
+ 'Lon[°]',
+ 'Lat[°]',
+ 'Altitude[m]',
+ 'GeoPot[m]',
+ 'MRI',
+ 'RI',    
+ 'DewPoint[C]',
+ 'Virt. Temp[C]',
+ 'Rs[m/min]',
+ 'D[kg/m3]',
+ 'Azimut[°]',
+ 'Elevation[°]',
+ 'Range[m]',
+]
+
+DTSTART = dt.datetime(2014,9,1,0,0,0,0,pytz.UTC)
+DTEND = dt.datetime(2014,10,1,0,0,0,0,pytz.UTC)
+
+
+DTS = [DTSTART+dt.timedelta(days=day) for day in range(0, int((DTEND-DTSTART).total_seconds()/3600./24.))]
+HOUR_FILES = {}
+for iDT, DT in enumerate(DTS):
+    morning_file = None
+    possible_files_morning =\
+    glob.glob(path_soundings_in+'/maosondewnpnM1.b1.'+DT.strftime("%Y%m%d")+'.23??00.*cdf')
+    if len(possible_files_morning)>0:
+        ix = 0
+        while ((ix < (len(possible_files_morning))) and (morning_file is None)):
+            # print (xr.open_dataset(possible_files_morning[ix]).pres.shape)
+            # print(xr.open_dataset(possible_files_morning[ix]).pres.shape[0] >= 10)
+            if (xr.open_dataset(possible_files_morning[ix]).pres.shape[0] >= 500) :
+            # and\
+            #   (possible_files_morning[ix] != '/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/GOAMAZON/Radiosounding_ARM/maosondewnpnM1.b1.20150103.115200.custom.cdf'):
+                morning_file = possible_files_morning[ix]
+            ix +=1
+
+
+    if (morning_file is not None):
+        HOUR_FILES[DT] = {'morning':  [int(morning_file.split('/')[-1].split('.')[3])/10000.,morning_file], }
+        print(HOUR_FILES[DT])
+
+# HOUR_FILES = \
+# {
+#     dt.datetime(2015,5,7,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150507.052900.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150507.172700.custom.cdf']},
+#     dt.datetime(2015,3,13,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150313.052700.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150313.173000.custom.cdf']},
+#     dt.datetime(2015,3,12,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150312.052800.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150312.173400.custom.cdf']},
+#     dt.datetime(2015,3,12,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150312.052800.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150312.173400.custom.cdf']},
+#     dt.datetime(2015,3,12,0,0,0,0,pytz.UTC):{'morning':  [5.5,'maosondewnpnM1.b1.20150312.052800.custom.cdf'],
+#                                              'afternoon':[17.50,'maosondewnpnM1.b1.20150312.173400.custom.cdf']},
+# }
+
+
+
+
+#only include the following timeseries in the model output
+timeseries_only = \
+['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+ 'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+ 'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+ 'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+ 'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+def esat(T):
+    return 0.611e3 * np.exp(17.2694 * (T - 273.16) / (T - 35.86))
+def efrom_rh100_T(rh100,T):
+    return esat(T)*rh100/100.
+def qfrom_e_p(e,p):
+    return epsilon * e/(p - (1.-epsilon)*e)
+
+def humppa_parser(balloon_file,file_sounding,ldate,lhour,c4gli=None):
+        print(balloon_file)
+        
+        xrin = balloon_file
+        air_balloon = pd.DataFrame()
+
+        air_balloon['t'] = xrin.tdry.values+273.15
+        air_balloon['p'] = xrin.pres.values*100.
+        
+        air_balloon['u'] = xrin.u_wind.values
+        air_balloon['v'] = xrin.v_wind.values
+        air_balloon['WSPD'] = xrin['wspd'].values
+        
+        print(xrin.rh.values.shape)
+        air_balloon['q'] = qfrom_e_p(efrom_rh100_T(xrin.rh.values,air_balloon['t'].values),air_balloon.p.values)
+        
+
+        #balloon_conv = replace_iter(balloon_file,"°","deg")
+        #readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()]
+        #air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15)
+        # air_balloon_in = pd.read_fwf(balloon_file,
+        #                              widths=[14]*19,
+        #                              skiprows=9,
+        #                              skipfooter=15,
+        #                              decimal=',',
+        #                              header=None,
+        #                              names = columns,
+        #                              na_values='-----')
+        
+        print(air_balloon['p'][0])
+        varcalc = {
+            'R' :    lambda x: (Rd*(1.-x.q) + Rv*x.q),
+            'pp': lambda x: (x['p'][0]/x['p'])**(x['R']/cp),
+            'theta': lambda x: (x['t']) * (x['p'][0]/x['p'])**(x['R']/cp),
+            'thetav': lambda x: x.theta  + 0.61 * x.theta * x.q,
+            'rho': lambda x: x.p /x.t / x.R ,
+        }
+        for varname,lfunction in varcalc.items():
+            air_balloon[varname] = lfunction(air_balloon)
+        
+        print('alt in xrin?:','alt' in xrin)
+        if 'alt' in xrin:
+            air_balloon['z'] = xrin.alt.values
+        else:
+            g          = 9.81                  # gravity acceleration [m s-2]
+            air_balloon['z'] = 0.
+            for irow,row in air_balloon.iloc[1:].iterrows():
+                air_balloon['z'].iloc[irow] = air_balloon['z'].iloc[irow-1] - \
+                        2./(air_balloon['rho'].iloc[irow-1]+air_balloon['rho'].iloc[irow])/g * \
+                        (air_balloon['p'].iloc[irow] - air_balloon['p'].iloc[irow-1])
+             
+        # for varname,lfunction in varcakc.items():
+        #     air_balloon[varname] = lfunction(air_balloon)
+        
+        dpars = {}
+        dpars['longitude']  = current_station['longitude']
+        dpars['latitude']  = current_station['latitude'] 
+        
+        dpars['STNID'] = current_station.name
+
+        # # there are issues with the lower measurements in the HUMPPA campaign,
+        # # for which a steady decrease of potential temperature is found, which
+        # # is unrealistic.  Here I filter them away
+        # ifirst = 0
+        # while  (air_balloon.theta.iloc[ifirst+1] < air_balloon.theta.iloc[ifirst]):
+        #     ifirst = ifirst+1
+        # print ('ifirst:',ifirst)
+        # air_balloon = air_balloon.iloc[ifirst:].reset_index().drop(['index'],axis=1)
+        air_balloon = air_balloon.iloc[:].reset_index().drop(['index'],axis=1)
+        
+        # if air_balloon.z.max() > 100000.:
+        #     air_balloon.z = air_balloon.z/10.
+
+        is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)
+        valid_indices = air_balloon.index[is_valid].values
+        
+        air_ap_mode='b'
+
+
+        while ((len(valid_indices) > 10) and
+               ((air_balloon.theta.iloc[valid_indices[0]] -
+                 air_balloon.theta.iloc[valid_indices[1]]) > 0.5)):
+            valid_indices = valid_indices[1:]
+
+        #theta_vs_first_inconsistent = True
+        # while theta_vs_first_inconsistent:
+        
+        if len(valid_indices) > 0:
+            air_balloon_temp = air_balloon.iloc[valid_indices]
+            print(air_balloon_temp)
+            print(air_balloon_temp.z.shape,air_balloon_temp.thetav.shape,)
+            dpars['h'],dpars['h_u'],dpars['h_l'] =\
+                blh(air_balloon_temp.z.values,air_balloon_temp.thetav.values,air_balloon_temp.WSPD.values)
+            dpars['h_b'] = np.max((dpars['h'],10.))
+            dpars['h_u'] = np.max((dpars['h_u'],10.)) #upper limit of mixed layer height
+            dpars['h_l'] = np.max((dpars['h_l'],10.)) #low limit of mixed layer height
+            dpars['h_e'] = np.abs( dpars['h_u'] - dpars['h_l']) # error of mixed-layer height
+            dpars['h'] = np.round(dpars['h_'+air_ap_mode],1)
+        else:
+            dpars['h_u'] =np.nan
+            dpars['h_l'] =np.nan
+            dpars['h_e'] =np.nan
+            dpars['h'] =np.nan
+        
+        if ~np.isnan(dpars['h']):
+            dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]]
+        else:
+            dpars['Ps'] = np.nan
+        
+        if ~np.isnan(dpars['h']):
+        
+            # determine mixed-layer properties (moisture, potential temperature...) from profile
+            
+            # ... and those of the mixed layer
+            is_valid_below_h = (air_balloon.iloc[valid_indices].z < dpars['h'])
+            valid_indices_below_h =  air_balloon.iloc[valid_indices].index[is_valid_below_h].values
+            if len(valid_indices) > 1:
+                if len(valid_indices_below_h) >= 3.:
+                    ml_mean = air_balloon.iloc[valid_indices][is_valid_below_h].mean()
+                else:
+                    ml_mean = air_balloon.iloc[valid_indices[0]:valid_indices[1]].mean()
+            elif len(valid_indices) == 1:
+                ml_mean = (air_balloon.iloc[0:1]).mean()
+            else:
+                temp =  pd.DataFrame(air_balloon)
+                temp.iloc[0] = np.nan
+                ml_mean = temp
+
+            dpars['theta']= ml_mean.theta
+            dpars['q']    = ml_mean.q
+            dpars['u']    = ml_mean.u
+            dpars['v']    = ml_mean.v 
+            # theta_vs_first_inconsistent = \
+            #     ((air_balloon.theta.iloc[valid_indices[0]] - air_balloon.theta.iloc[valid_indices[1]]) > 0.2)
+            # theta_vs_first_inconsistent = \
+            #     ((air_balloon.theta.iloc[valid_indices[0]] - dpars['theta']) > 0.1)
+            # if theta_vs_first_inconsistent:
+            #     valid_indices = valid_indices[1:]
+            #     print("warning! too large difference between near surface value and abl value of theta. I'm taking the next one as near surface vlue")
+        else:
+            dpars['theta'] = np.nan
+            dpars['q'] = np.nan
+            dpars['u'] = np.nan
+            dpars['v'] = np.nan
+            # theta_bl_inconsistent = False
+
+        air_ap_head = air_balloon[0:0] #pd.DataFrame(columns = air_balloon.columns)
+        # All other  data points above the mixed-layer fit
+        air_ap_tail = air_balloon[air_balloon.z > dpars['h']]
+
+        air_ap_head.z = pd.Series(np.array([2.,dpars['h'],dpars['h']]))
+        jump = air_ap_head.iloc[0] * np.nan
+        
+        if air_ap_tail.shape[0] > 1:
+        
+            # we originally used THTA, but that has another definition than the
+            # variable theta that we need which should be the temperature that
+            # one would have if brought to surface (NOT reference) pressure.
+            for column in ['theta','q','u','v']:
+               
+               # initialize the profile head with the mixed-layer values
+               air_ap_head[column] = ml_mean[column]
+               # calculate jump values at mixed-layer height, which will be
+               # added to the third datapoint of the profile head
+               jump[column] = (air_ap_tail[column].iloc[1]\
+                               -\
+                               air_ap_tail[column].iloc[0])\
+                              /\
+                              (air_ap_tail.z.iloc[1]\
+                               - air_ap_tail.z.iloc[0])\
+                              *\
+                              (dpars['h']- air_ap_tail.z.iloc[0])\
+                              +\
+                              air_ap_tail[column].iloc[0]\
+                              -\
+                              ml_mean[column] 
+               if column == 'theta':
+                  # for potential temperature, we need to set a lower limit to
+                  # avoid the model to crash
+                  jump.theta = np.max((0.1,jump.theta))
+        
+               air_ap_head[column][2] += jump[column]
+        
+        air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 +air_ap_head.v**2)
+
+        # only select samples monotonically increasing with height
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        print(air_ap_tail_orig)
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        for ibottom in range(1,len(air_ap_tail_orig)):
+            if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z +10.:
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],ignore_index=True)
+
+        # make theta increase strong enough to avoid numerical
+        # instability
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        # air_ap_tail = pd.DataFrame()
+        # #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # theta_low = air_ap_head['theta'].iloc[2]
+        # z_low = air_ap_head['z'].iloc[2]
+        # ibottom = 0
+        # for itop in range(0,len(air_ap_tail_orig)):
+        #     theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+        #     z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+        #     if (
+        #         #(z_mean > z_low) and \
+        #         (z_mean > (z_low+10.)) and \
+        #         #(theta_mean > (theta_low+0.2) ) and \
+        #         #(theta_mean > (theta_low+0.2) ) and \
+        #          (((theta_mean - theta_low)/(z_mean - z_low)) > 0.00001)):
+
+        #         air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+        #         ibottom = itop+1
+        #         theta_low = air_ap_tail.theta.iloc[-1]
+        #         z_low =     air_ap_tail.z.iloc[-1]
+        #     # elif  (itop > len(air_ap_tail_orig)-10):
+        #     #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        # 
+        air_ap = \
+            pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+        
+        # we copy the pressure at ground level from balloon sounding. The
+        # pressure at mixed-layer height will be determined internally by class
+        
+        rho        = 1.2                   # density of air [kg m-3]
+        g          = 9.81                  # gravity acceleration [m s-2]
+        
+        air_ap['p'].iloc[0] =dpars['Ps'] 
+        air_ap['p'].iloc[1] =(dpars['Ps'] - rho * g * dpars['h'])
+        air_ap['p'].iloc[2] =(dpars['Ps'] - rho * g * dpars['h'] -0.1)
+        
+        dpars['lat'] = dpars['latitude']
+        # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich)
+        dpars['lon'] = 0.
+        # this is the real longitude that will be used to extract ground data
+        
+        # dpars['ldatetime'] = ldate+dt.timedelta(hours=lhour)
+        # dpars['datetime'] =  dpars['ldatetime'] + dt.timedelta(hours=+4)
+
+        dpars['datetime'] =  ldate+dt.timedelta(hours=lhour)
+        dpars['ldatetime'] =  dpars['datetime'] + dt.timedelta(hours=-4)
+
+
+
+
+        dpars['doy'] = dpars['datetime'].timetuple().tm_yday
+        
+        dpars['SolarAltitude'] = \
+                                Pysolar.GetAltitude(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        dpars['SolarAzimuth'] =  Pysolar.GetAzimuth(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        
+        dpars['lSunrise'], dpars['lSunset'] \
+        =  Pysolar.util.GetSunriseSunset(dpars['latitude'],
+                                         0.,
+                                         dpars['ldatetime'],0.)
+        
+        # Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually write local solar time, we need to assign the timezone to UTC (which is WRONG!!!). Otherwise ruby cannot understand it (it always converts tolocal computer time :( ). 
+        dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise'])
+        dpars['lSunset'] = pytz.utc.localize(dpars['lSunset'])
+        
+        # This is the nearest datetime when the sun is up (for class)
+        dpars['ldatetime_daylight'] = \
+                                np.min(\
+                                    (np.max(\
+                                        (dpars['ldatetime'],\
+                                         dpars['lSunrise'])\
+                                     ),\
+                                     dpars['lSunset']\
+                                    )\
+                                )
+        # apply the same time shift for UTC datetime
+        dpars['datetime_daylight'] = dpars['datetime'] \
+                                    +\
+                                    (dpars['ldatetime_daylight']\
+                                     -\
+                                     dpars['ldatetime'])
+        
+        
+        # We set the starting time to the local sun time, since the model 
+        # thinks we are always at the meridian (lon=0). This way the solar
+        # radiation is calculated correctly.
+        dpars['tstart'] = dpars['ldatetime_daylight'].hour \
+                         + \
+                         dpars['ldatetime_daylight'].minute/60.\
+                         + \
+                         dpars['ldatetime_daylight'].second/3600.
+        
+        dpars['sw_lit'] = False
+        # convert numpy types to native python data types. This provides
+        # cleaner data IO with yaml:
+        for key,value in dpars.items():
+            if type(value).__module__ == 'numpy':
+                dpars[key] = dpars[key].item()
+        
+                decimals = {'p':0,'t':2,'theta':4, 'z':2, 'q':5, 'u':4, 'v':4}
+        # 
+                for column,decimal in decimals.items():
+                    air_balloon[column] = air_balloon[column].round(decimal)
+                    air_ap[column] = air_ap[column].round(decimal)
+        
+
+
+        dpars['gammatheta_lower_limit'] = 0.0001
+        updateglobal = False
+        if c4gli is None:
+            c4gli = class4gl_input()
+            updateglobal = True
+        
+        print('updating...')
+        print(column)
+        c4gli.update(source='goamazon',\
+                    # pars=pars,
+                    pars=dpars,\
+                    air_balloon=air_balloon,\
+                    air_ap=air_ap)
+        if updateglobal:
+            c4gli.get_global_input(globaldata)
+
+        # if profile_ini:
+        #     c4gli.runtime = 10 * 3600
+
+        if not ((dpars['ldatetime'].hour <=12) or\
+           ((dpars['lSunset'].hour - dpars['ldatetime'].hour) >= (2.))):
+            c4gli = None
+        
+        # if profile_ini:
+        #     c4gl = class4gl(c4gli)
+        #     c4gl.run()
+        #     c4gl.dump(file_model,\
+        #               include_input=True,\
+        #               timeseries_only=timeseries_only)
+        #     
+        #     # This will cash the observations and model tables per station for
+        #     # the interface
+        # 
+        # if profile_ini:
+        #     profile_ini=False
+        # else:
+        #     profile_ini=True
+        return c4gli
+
+
+# path_soundings = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/IOPS/'
+path_soundings = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/GOAMAZON_EVENING/'
+os.system('mkdir -p '+path_soundings)
+
+file_morning = open(path_soundings+format(current_station.name,'05d')+'_ini.yaml','w') 
+ipair = 0
+for date,pair  in HOUR_FILES.items(): 
+    
+    humpafn = pair['morning'][1]
+    balloon_file_morning = xr.open_dataset(humpafn)
+    if (balloon_file_morning.pres.shape[0] > 10):
+        c4gli_morning = humppa_parser(balloon_file_morning,file_morning,date,pair['morning'][0])
+        if (c4gli_morning is not None) :
+            c4gli_morning.dump(file_morning)
+        ipair += 1
+
+print(ipair)
+file_morning.close()
+
+ 
+
+# file_morning = open(path_soundings+format(current_station.name,'05d')+'_morning.yaml','w') 
+# for date,pair  in HOUR_FILES.items(): 
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/GOAMAZON/Radiosounding_ARM/'+pair['morning'][1],
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_morning,hour,c4gli_morning)
+#     print('c4gli_morning_ldatetime 1',c4gli_morning.pars.ldatetime)
+# file_morning.close()
+# 
+# file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_afternoon.yaml','w') 
+# for hour in [18]:
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/GOAMAZON/Radiosounding_ARM//humppa_080610_'+format(hour,"02d")+'00.txt'
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_afternoon,hour,c4gli_afternoon)
+# file_afternoon.close()
+
+
+
+# path_model = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/HUMPPA/'
+# 
+# file_model    = open(fnout_model+   format(current_station.name,'05d')+'.yaml','w') 
+
+records_morning = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='ini',\
+                                           refetch_records=True,\
+                                           )
+
+# align afternoon records with noon records, and set same index
+
+# os.system('mkdir -p '+path_exp)
+# file_morning = open(path_soundings+'/'+format(current_station.name,'05d')+'_ini.yaml')
+# file_afternoon = open(path_soundings+'/'+format(current_station.name,'05d')+'_end.yaml')
+# file_ini = open(path_exp+'/'+format(current_station.name,'05d')+'_ini.yaml','w')
+# file_mod = open(path_exp+'/'+format(current_station.name,'05d')+'_mod.yaml','w')
+# 
+# for (STNID,chunk,index),record_morning in records_morning.iterrows():
+#     record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+# 
+#     c4gli_morning = get_record_yaml(file_morning, 
+#                                     record_morning.index_start, 
+#                                     record_morning.index_end,
+#                                     mode='ini')
+#     #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+#     
+#     
+#     c4gli_afternoon = get_record_yaml(file_afternoon, 
+#                                       record_afternoon.index_start, 
+#                                       record_afternoon.index_end,
+#                                     mode='ini')
+# 
+#     c4gli_morning.update(source='pairs',pars={'runtime' : \
+#                         int((c4gli_afternoon.pars.datetime_daylight - 
+#                              c4gli_morning.pars.datetime_daylight).total_seconds())})
+#     c4gli_morning.update(source='manual',
+#                          pars={'sw_ac' : [],'sw_ap': True,'sw_lit': False})
+#     c4gli_morning.dump(file_ini)
+#     
+#     c4gl = class4gl(c4gli_morning)
+#     c4gl.run()
+#     
+#     c4gl.dump(file_mod,\
+#               include_input=False,\
+#               timeseries_only=timeseries_only)
+# file_ini.close()
+# file_mod.close()
+# file_morning.close()
+# file_afternoon.close()
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                            path_exp,\
+#                                            subset='ini',
+#                                            refetch_records=True,
+#                                            )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                            path_exp,\
+#                                            subset='mod',
+#                                            refetch_records=True,
+#                                            )
+# 
+# records_mod.index = records_ini.index
+# 
+# # align afternoon records with initial records, and set same index
+# records_afternoon.index = records_afternoon.ldatetime.dt.date
+# records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+# records_afternoon.index = records_ini.index
+
+
+"""
+stations_for_iter = stations(path_exp)
+for STNID,station in stations_iterator(stations_for_iter):
+    records_current_station_index = \
+            (records_ini.index.get_level_values('STNID') == STNID)
+    file_current_station_mod = STNID
+
+    with \
+    open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+    open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+        for (STNID,index),record_ini in records_iterator(records_ini):
+            c4gli_ini = get_record_yaml(file_station_ini, 
+                                        record_ini.index_start, 
+                                        record_ini.index_end,
+                                        mode='ini')
+            #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+
+            record_mod = records_mod.loc[(STNID,index)]
+            c4gl_mod = get_record_yaml(file_station_mod, 
+                                        record_mod.index_start, 
+                                        record_mod.index_end,
+                                        mode='mod')
+            record_afternoon = records_afternoon.loc[(STNID,index)]
+            c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+                                        record_afternoon.index_start, 
+                                        record_afternoon.index_end,
+                                        mode='ini')
+"""
+
+
+# # select the samples of the afternoon list that correspond to the timing of the
+# # morning list
+# records_afternoon = records_afternoon.set_index('ldatetime').loc[records_afternoon.ldatetime)]
+# records_afternoon.index = recods_morning.index
+# 
+# 
+# # create intersectino index
+# index_morning = pd.Index(records_morning.ldatetime.to_date())
+# index_afternoon = pd.Index(records_afternoon.ldatetime.to_date())
+# 
+# for record_morning in records_morning.iterrows():
+#     
+#     c4gl = class4gl(c4gli)
+#     c4gl.run()
+#     c4gl.dump(c4glfile,\
+#               include_input=True,\
+#               timeseries_only=timeseries_only)
+# 
+# # This will cash the observations and model tables per station for
+# # the interface
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=0,\
+#                                    by=2,\
+#                                    subset='ini',
+#                                    refetch_records=True,
+#                                    )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='mod',
+#                                    refetch_records=True,
+#                                    )
+# records_eval = get_records(pd.DataFrame([current_station]),\
+#                                    path_obs,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='eval',
+#                                    refetch_records=True,
+#                                    )
+# 
+# 
+# # mod_scores = pd.DataFrame(index=mod_records.index)
+# # for (STNID,index), current_record_mod in mod_records.iterrows():
+# #     print(STNID,index)
+# #     current_station = STN
+# #     current_record_obs_afternoon = obs_records_afternoon.loc[(STNID,index)]
+# #     current_record_obs = obs_records.loc[(STNID,index)]
+# # 
+# #     record_yaml_mod = get_record_yaml_mod(odirexperiments[keyEXP],\
+# #                                           current_station,\
+# #                                           current_record_mod,\
+# #                                          )
+# # 
+# #     record_yaml_obs = \
+# #             get_record_yaml_obs(odirexperiments[keyEXP],\
+# #                                 current_station,\
+# #                                 current_record_obs,\
+# #                                 suffix='.yaml')
+# # 
+# #     record_yaml_obs_afternoon = \
+# #             get_record_yaml_obs(odir,\
+# #                                 current_station,\
+# #                                 current_record_obs_afternoon,\
+# #                                 suffix='_afternoon.yaml')
+# # 
+# #     hmax = np.max([record_yaml_obs_afternoon.pars.h,\
+# #                    record_yaml_mod.h])
+# #     HEIGHTS = {'h':hmax, '2h':2.*hmax, '3000m':3000.}
+# #     
+# # 
+# #     for height,hvalue in HEIGHTS.items():
+# # 
+# #         lt_obs = (record_yaml_obs_afternoon.air_ap.HAGL < hvalue)
+# #         lt_mod = (record_yaml_mod.air_ap.z < hvalue)
+# #         try:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = \
+# #                 rmse(\
+# #                     record_yaml_obs_afternoon.air_ap.theta[lt_obs],\
+# #                     np.interp(\
+# #                         record_yaml_obs_afternoon.air_ap.HAGL[lt_obs],\
+# #                         record_yaml_mod.air_ap.z[lt_mod],\
+# #                         record_yaml_mod.air_ap.theta[lt_mod]\
+# #                     ))
+# #         except ValueError:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = np.nan
+# #     # # we calculate these things in the interface itself
+# #     # for key in ['q','theta','h']:
+# #     #     mod_records.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_mod.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# #     #     # the actual time of the initial and evaluation sounding can be 
+# #     #     # different, but we consider this as a measurement error for
+# #     #     # the starting and end time of the simulation.
+# #     #     obs_records_afternoon.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_obs_afternoon.pars.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# # mod_scores.to_pickle(odirexperiments[keyEXP]+'/'+format(STNID,'05d')+"_mod_scores.pkl")
+# #         
+# #                 
+# #                 
+# # # for EXP,c4glfile in c4glfiles.items():
+# # #     c4glfile.close()            
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# #     
+# #     # {'Time[min:sec]': None 
+# #     #  'P[hPa]': None, 
+# #     #  'T[C]': None, 
+# #     #  'U[%]': None, 
+# #     #  'Wsp[m/s]': None, 
+# #     #  'Wdir[Grd]': None,
+# #     #  'Lon[°]', 
+# #     #  'Lat[°]', 
+# #     #  'Altitude[m]', 'GeoPot[m']', 'MRI',
+# #     #        'Unnamed: 11', 'RI', 'Unnamed: 13', 'DewPoint[C]', 'Virt. Temp[C]',
+# #     #        'Rs[m/min]D[kg/m3]Azimut[deg]', 'Elevation[deg]', 'Range[m]']
+# #     # }
+# #     # 
+# #     # #pivotrows =
+# #     # #{
+# # 
+# # 
+# # 
diff --git a/class4gl/setup/setup_humppa.py b/class4gl/setup/setup_humppa.py
new file mode 100644
index 0000000..273e039
--- /dev/null
+++ b/class4gl/setup/setup_humppa.py
@@ -0,0 +1,740 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import Pysolar
+import sys
+import pytz
+sys.path.insert(0,'/user/home/gent/vsc422/vsc42247/software/class4gl/class4gl/')
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+
+globaldata = data_global()
+globaldata.load_datasets(recalc=0)
+
+Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+epsilon = Rd/Rv # or mv/md
+
+
+def replace_iter(iterable, search, replace):
+    for value in iterable:
+        value.replace(search, replace)
+        yield value
+
+from class4gl import blh,class4gl_input
+
+# definition of the humpa station
+current_station = pd.Series({ "latitude"  : 61.8448,
+                  "longitude" : 24.2882,
+                  "name" : "the HUMMPA experiment"
+                })
+current_station.name = 90000
+
+# we define the columns ourselves because it is a mess in the file itself.
+columns =\
+['Time[min:sec]',
+ 'P[hPa]',
+ 'T[C]',
+ 'U[%]',
+ 'Wsp[m/s]',
+ 'Wdir[Grd]',
+ 'Lon[°]',
+ 'Lat[°]',
+ 'Altitude[m]',
+ 'GeoPot[m]',
+ 'MRI',
+ 'RI',    
+ 'DewPoint[C]',
+ 'Virt. Temp[C]',
+ 'Rs[m/min]',
+ 'D[kg/m3]',
+ 'Azimut[°]',
+ 'Elevation[°]',
+ 'Range[m]',
+]
+
+
+HOUR_FILES = \
+{ dt.datetime(2010,7,12,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071210_0300.txt'],'afternoon':[15,'humppa_071210_1500.txt']},
+  dt.datetime(2010,7,13,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071310_0300.txt'],'afternoon':[18,'humppa_071310_1800.txt']},
+  dt.datetime(2010,7,14,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071410_0300.txt'],'afternoon':[16,'humppa_071410_1600.txt']},
+  dt.datetime(2010,7,15,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071510_0300.txt'],'afternoon':[15,'humppa_071510_1500.txt']},
+  dt.datetime(2010,7,16,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071610_0300.txt'],'afternoon':[21,'humppa_071610_2100.txt']},
+  dt.datetime(2010,7,17,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071710_0300.txt'],'afternoon':[18,'humppa_071710_1800.txt']},
+  dt.datetime(2010,7,18,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071810_0300.txt'],'afternoon':[21,'humppa_071810_2100.txt']},
+  dt.datetime(2010,7,19,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071910_0300.txt'],'afternoon':[21,'humppa_071910_2100.txt']},
+#  dt.datetime(2010,7,20):{'morning':[4,'humppa_072010_0400.txt'],'afternoon':[15,'humppa_072010_1500.txt']},
+  dt.datetime(2010,7,21,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072110_0300.txt'],'afternoon':[21,'humppa_072110_2100.txt']},
+  dt.datetime(2010,7,22,0,0,0,0,pytz.UTC):{'morning':[4,'humppa_072210_0400.txt'],'afternoon':[18,'humppa_072210_1800.txt']},
+ # something is wrong with ths profile
+ # dt.datetime(2010,7,23,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072310_0300.txt'],'afternoon':[15,'humppa_072310_1500.txt']},
+  dt.datetime(2010,7,24,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072410_0300.txt'],'afternoon':[16,'humppa_072410_1600.txt']},
+  dt.datetime(2010,7,25,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072510_0300.txt'],'afternoon':[21,'humppa_072510_2100.txt']},
+  dt.datetime(2010,7,26,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072610_0300.txt'],'afternoon':[21,'humppa_072610_2100.txt']},
+  dt.datetime(2010,7,27,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072710_0300.txt'],'afternoon':[15,'humppa_072710_1500.txt']},
+  dt.datetime(2010,7,28,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072810_0300.txt'],'afternoon':[15,'humppa_072810_1500.txt']},
+  dt.datetime(2010,7,29,0,0,0,0,pytz.UTC):{'morning':[4,'humppa_072910_0400.txt'],'afternoon':[18,'humppa_072910_1800.txt']},
+  dt.datetime(2010,7,30,0,0,0,0,pytz.UTC):{'morning':[9,'humppa_073010_0900.txt'],'afternoon':[15,'humppa_073010_1500.txt']},
+  dt.datetime(2010,7,31,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_073110_0300_01.txt'],'afternoon':[15,'humppa_073110_1500.txt']},
+  dt.datetime(2010,8, 1,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080110_0300.txt'],'afternoon':[18,'humppa_080110_1800.txt']},
+  dt.datetime(2010,8, 2,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080210_0300.txt'],'afternoon':[18,'humppa_080210_1800.txt']},
+  dt.datetime(2010,8, 3,0,0,0,0,pytz.UTC):{'morning':[9,'humppa_080310_0900.txt'],'afternoon':[18,'humppa_080310_1800.txt']},
+  dt.datetime(2010,8, 3,0,0,0,0,pytz.UTC):{'morning':[8,'humppa_080410_0800.txt'],'afternoon':[18,'humppa_080410_1800.txt']},
+  dt.datetime(2010,8, 5,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080510_0300.txt'],'afternoon':[18,'humppa_080510_1800.txt']},
+  dt.datetime(2010,8, 6,0,0,0,0,pytz.UTC):{'morning':[4,'humppa_080610_0400.txt'],'afternoon':[18,'humppa_080610_1800.txt']},
+  dt.datetime(2010,8, 7,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080710_0300.txt'],'afternoon':[18,'humppa_080710_1800.txt']},
+  dt.datetime(2010,8, 8,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080810_0300.txt'],'afternoon':[18,'humppa_080810_1800.txt']},
+  dt.datetime(2010,8,10,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_081010_0300.txt'],'afternoon':[18,'humppa_081010_1800.txt']},
+}
+
+
+
+
+
+
+#only include the following timeseries in the model output
+timeseries_only = \
+['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+ 'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+ 'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+ 'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+ 'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+def humppa_parser(balloon_file,file_sounding,ldate,hour,c4gli=None):
+        #balloon_conv = replace_iter(balloon_file,"°","deg")
+        #readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()]
+        #air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15)
+        air_balloon_in = pd.read_fwf(balloon_file,
+                                     widths=[14]*19,
+                                     skiprows=9,
+                                     skipfooter=15,
+                                     decimal=',',
+                                     header=None,
+                                     names = columns,
+                                     na_values='-----')
+    
+        rowmatches = {
+            't':      lambda x: x['T[C]']+273.15,
+            #'tv':     lambda x: x['Virt. Temp[C]']+273.15,
+            'p':      lambda x: x['P[hPa]']*100.,
+            'u':      lambda x: x['Wsp[m/s]'] * np.sin((90.-x['Wdir[Grd]'])/180.*np.pi),
+            'v':      lambda x: x['Wsp[m/s]'] * np.cos((90.-x['Wdir[Grd]'])/180.*np.pi),
+            'z':      lambda x: x['Altitude[m]'],
+            'q':      lambda x: np.clip((1. - (273.15+x['Virt. Temp[C]'])/(273.15+x['T[C]']))/(1. - 1./epsilon),a_min=0.,a_max=None),
+        }
+        
+        air_balloon = pd.DataFrame()
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon_in)
+        
+        rowmatches = {
+            'R' :    lambda x: (Rd*(1.-x.q) + Rv*x.q),
+            'theta': lambda x: (x['t']) * (x['p'][0]/x['p'])**(x['R']/cp),
+            'thetav': lambda x: x.theta  + 0.61 * x.theta * x.q
+        }
+        
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon)
+        
+        dpars = {}
+        dpars['longitude']  = current_station['longitude']
+        dpars['latitude']  = current_station['latitude'] 
+        
+        dpars['STNID'] = current_station.name
+        
+
+        # there are issues with the lower measurements in the HUMPPA campaign,
+        # for which a steady decrease of potential temperature is found, which
+        # is unrealistic.  Here I filter them away
+        ifirst = 0
+        while  (air_balloon.theta.iloc[ifirst+1] < air_balloon.theta.iloc[ifirst]):
+            ifirst = ifirst+1
+        print ('ifirst:',ifirst)
+        air_balloon = air_balloon.iloc[ifirst:].reset_index().drop(['index'],axis=1)
+        
+        is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)
+        valid_indices = air_balloon.index[is_valid].values
+        i = 1
+        while (air_balloon.thetav.iloc[valid_indices[0]] - \
+               air_balloon.thetav.iloc[valid_indices[i]] ) > 0.5:
+            #diff = (air_balloon.theta.iloc[valid_indices[i]] -air_balloon.theta.iloc[valid_indices[i+1]])- 0.5
+            air_balloon.thetav.iloc[valid_indices[0:i]] = \
+                air_balloon.thetav.iloc[valid_indices[i]] + 0.5 
+            
+            i +=1
+        
+        air_ap_mode='b'
+        
+        if len(valid_indices) > 0:
+            dpars['h'],dpars['h_u'],dpars['h_l'] =\
+                blh(air_balloon.z,air_balloon.thetav,air_balloon_in['Wsp[m/s]'])
+            dpars['h_b'] = np.max((dpars['h'],10.))
+            dpars['h_u'] = np.max((dpars['h_u'],10.)) #upper limit of mixed layer height
+            dpars['h_l'] = np.max((dpars['h_l'],10.)) #low limit of mixed layer height
+            dpars['h_e'] = np.abs( dpars['h_u'] - dpars['h_l']) # error of mixed-layer height
+            dpars['h'] = np.round(dpars['h_'+air_ap_mode],1)
+        else:
+            dpars['h_u'] =np.nan
+            dpars['h_l'] =np.nan
+            dpars['h_e'] =np.nan
+            dpars['h'] =np.nan
+        
+        
+        
+        if ~np.isnan(dpars['h']):
+            dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]]
+        else:
+            dpars['Ps'] = np.nan
+        
+        if ~np.isnan(dpars['h']):
+        
+            # determine mixed-layer properties (moisture, potential temperature...) from profile
+            
+            # ... and those of the mixed layer
+            is_valid_below_h = (air_balloon.iloc[valid_indices].z < dpars['h'])
+            valid_indices_below_h =  air_balloon.iloc[valid_indices].index[is_valid_below_h].values
+            if len(valid_indices) > 1:
+                if len(valid_indices_below_h) >= 3.:
+                    ml_mean = air_balloon.iloc[valid_indices][is_valid_below_h].mean()
+                else:
+                    ml_mean = air_balloon.iloc[valid_indices[0]:valid_indices[1]].mean()
+            elif len(valid_indices) == 1:
+                ml_mean = (air_balloon.iloc[0:1]).mean()
+            else:
+                temp =  pd.DataFrame(air_balloon)
+                temp.iloc[0] = np.nan
+                ml_mean = temp
+                      
+            dpars['theta']= ml_mean.theta
+            dpars['q']    = ml_mean.q
+            dpars['u']    = ml_mean.u
+            dpars['v']    = ml_mean.v 
+        else:
+            dpars['theta'] = np.nan
+            dpars['q'] = np.nan
+            dpars['u'] = np.nan
+            dpars['v'] = np.nan
+        
+        air_ap_head = air_balloon[0:0] #pd.DataFrame(columns = air_balloon.columns)
+        # All other  data points above the mixed-layer fit
+        air_ap_tail = air_balloon[air_balloon.z > dpars['h']]
+
+
+
+        air_ap_head.z = pd.Series(np.array([2.,dpars['h'],dpars['h']]))
+        jump = air_ap_head.iloc[0] * np.nan
+        
+        if air_ap_tail.shape[0] > 1:
+        
+            # we originally used THTA, but that has another definition than the
+            # variable theta that we need which should be the temperature that
+            # one would have if brought to surface (NOT reference) pressure.
+            for column in ['theta','q','u','v']:
+               
+               # initialize the profile head with the mixed-layer values
+               air_ap_head[column] = ml_mean[column]
+               # calculate jump values at mixed-layer height, which will be
+               # added to the third datapoint of the profile head
+               jump[column] = (air_ap_tail[column].iloc[1]\
+                               -\
+                               air_ap_tail[column].iloc[0])\
+                              /\
+                              (air_ap_tail.z.iloc[1]\
+                               - air_ap_tail.z.iloc[0])\
+                              *\
+                              (dpars['h']- air_ap_tail.z.iloc[0])\
+                              +\
+                              air_ap_tail[column].iloc[0]\
+                              -\
+                              ml_mean[column] 
+               if column == 'theta':
+                  # for potential temperature, we need to set a lower limit to
+                  # avoid the model to crash
+                  jump.theta = np.max((0.1,jump.theta))
+        
+               air_ap_head[column][2] += jump[column]
+        
+        air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 +air_ap_head.v**2)
+
+
+
+        # only select samples monotonically increasing with height
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        for ibottom in range(1,len(air_ap_tail_orig)):
+            if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z +10.:
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],ignore_index=True)
+
+        # make theta increase strong enough to avoid numerical
+        # instability
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        theta_low = air_ap_head['theta'].iloc[2]
+        z_low = air_ap_head['z'].iloc[2]
+        ibottom = 0
+        for itop in range(0,len(air_ap_tail_orig)):
+            theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+            z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+            if (
+                #(z_mean > z_low) and \
+                (z_mean > (z_low+10.)) and \
+                #(theta_mean > (theta_low+0.2) ) and \
+                #(theta_mean > (theta_low+0.2) ) and \
+                 (((theta_mean - theta_low)/(z_mean - z_low)) > 0.0001)):
+
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+                ibottom = itop+1
+                theta_low = air_ap_tail.theta.iloc[-1]
+                z_low =     air_ap_tail.z.iloc[-1]
+            # elif  (itop > len(air_ap_tail_orig)-10):
+            #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        
+        air_ap = \
+            pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+
+        # # make theta increase strong enough to avoid numerical
+        # # instability
+        # air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        # air_ap_tail = pd.DataFrame()
+        # #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # theta_low = air_ap_head['theta'].iloc[2]
+        # z_low = air_ap_head['z'].iloc[2]
+        # ibottom = 0
+        # for itop in range(0,len(air_ap_tail_orig)):
+        #     theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+        #     z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+        #     if ((theta_mean > (theta_low+0.2) ) and \
+        #          (((theta_mean - theta_low)/(z_mean - z_low)) > 0.001)):
+
+        #         air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+        #         ibottom = itop+1
+        #         theta_low = air_ap_tail.theta.iloc[-1]
+        #         z_low =     air_ap_tail.z.iloc[-1]
+        #     # elif  (itop > len(air_ap_tail_orig)-10):
+        #     #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        # 
+        # air_ap = \
+        #     pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+        
+        # we copy the pressure at ground level from balloon sounding. The
+        # pressure at mixed-layer height will be determined internally by class
+        
+        rho        = 1.2                   # density of air [kg m-3]
+        g          = 9.81                  # gravity acceleration [m s-2]
+        
+        air_ap['p'].iloc[0] =dpars['Ps'] 
+        air_ap['p'].iloc[1] =(dpars['Ps'] - rho * g * dpars['h'])
+        air_ap['p'].iloc[2] =(dpars['Ps'] - rho * g * dpars['h'] -0.1)
+        
+        
+        dpars['lat'] = dpars['latitude']
+        # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich)
+        dpars['lon'] = 0.
+        # this is the real longitude that will be used to extract ground data
+        
+        dpars['ldatetime'] = ldate+dt.timedelta(hours=hour)
+        dpars['datetime'] =  dpars['ldatetime'] + dt.timedelta(hours=-3)
+        dpars['doy'] = dpars['datetime'].timetuple().tm_yday
+        
+        dpars['SolarAltitude'] = \
+                                Pysolar.GetAltitude(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        dpars['SolarAzimuth'] =  Pysolar.GetAzimuth(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        
+        
+        dpars['lSunrise'], dpars['lSunset'] \
+        =  Pysolar.util.GetSunriseSunset(dpars['latitude'],
+                                         0.,
+                                         dpars['ldatetime'],0.)
+        
+        # Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually write local solar time, we need to assign the timezone to UTC (which is WRONG!!!). Otherwise ruby cannot understand it (it always converts tolocal computer time :( ). 
+        dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise'])
+        dpars['lSunset'] = pytz.utc.localize(dpars['lSunset'])
+        
+        # This is the nearest datetime when the sun is up (for class)
+        dpars['ldatetime_daylight'] = \
+                                np.min(\
+                                    (np.max(\
+                                        (dpars['ldatetime'],\
+                                         dpars['lSunrise'])\
+                                     ),\
+                                     dpars['lSunset']\
+                                    )\
+                                )
+        # apply the same time shift for UTC datetime
+        dpars['datetime_daylight'] = dpars['datetime'] \
+                                    +\
+                                    (dpars['ldatetime_daylight']\
+                                     -\
+                                     dpars['ldatetime'])
+        
+        
+        # We set the starting time to the local sun time, since the model 
+        # thinks we are always at the meridian (lon=0). This way the solar
+        # radiation is calculated correctly.
+        dpars['tstart'] = dpars['ldatetime_daylight'].hour \
+                         + \
+                         dpars['ldatetime_daylight'].minute/60.\
+                         + \
+                         dpars['ldatetime_daylight'].second/3600.
+        
+        dpars['sw_lit'] = False
+        # convert numpy types to native python data types. This provides
+        # cleaner data IO with yaml:
+        for key,value in dpars.items():
+            if type(value).__module__ == 'numpy':
+                dpars[key] = dpars[key].item()
+        
+                decimals = {'p':0,'t':2,'theta':4, 'z':2, 'q':5, 'u':4, 'v':4}
+        # 
+                for column,decimal in decimals.items():
+                    air_balloon[column] = air_balloon[column].round(decimal)
+                    air_ap[column] = air_ap[column].round(decimal)
+        
+        updateglobal = False
+        if c4gli is None:
+            c4gli = class4gl_input()
+            updateglobal = True
+        
+        print('updating...')
+        print(column)
+        c4gli.update(source='humppa',\
+                    # pars=pars,
+                    pars=dpars,\
+                    air_balloon=air_balloon,\
+                    air_ap=air_ap)
+        if updateglobal:
+            c4gli.get_global_input(globaldata)
+
+        # if profile_ini:
+        #     c4gli.runtime = 10 * 3600
+
+        c4gli.dump(file_sounding)
+        
+        # if profile_ini:
+        #     c4gl = class4gl(c4gli)
+        #     c4gl.run()
+        #     c4gl.dump(file_model,\
+        #               include_input=True,\
+        #               timeseries_only=timeseries_only)
+        #     
+        #     # This will cash the observations and model tables per station for
+        #     # the interface
+        # 
+        # if profile_ini:
+        #     profile_ini=False
+        # else:
+        #     profile_ini=True
+        return c4gli
+
+
+path_soundings = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/IOPS/'
+
+
+file_morning = open(path_soundings+format(current_station.name,'05d')+'_ini.yaml','w') 
+for date,pair  in HOUR_FILES.items(): 
+    print(pair['morning'])
+    humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/'+pair['morning'][1]
+    print(humpafn)
+    balloon_file = open(humpafn,'r',encoding='latin-1')
+
+    c4gli_morning = humppa_parser(balloon_file,file_morning,date,pair['morning'][0])
+    print('c4gli_morning_ldatetime 0',c4gli_morning.pars.ldatetime)
+file_morning.close()
+
+file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_end.yaml','w') 
+for date,pair  in HOUR_FILES.items(): 
+    humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/'+pair['afternoon'][1]
+    balloon_file = open(humpafn,'r',encoding='latin-1')
+
+    c4gli_afternoon = humppa_parser(balloon_file,file_afternoon,date,pair['afternoon'][0])
+    print('c4gli_afternoon_ldatetime 0',c4gli_afternoon.pars.ldatetime)
+file_afternoon.close()
+ 
+
+# file_morning = open(path_soundings+format(current_station.name,'05d')+'_morning.yaml','w') 
+# for date,pair  in HOUR_FILES.items(): 
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/'+pair['morning'][1],
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_morning,hour,c4gli_morning)
+#     print('c4gli_morning_ldatetime 1',c4gli_morning.pars.ldatetime)
+# file_morning.close()
+# 
+# file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_afternoon.yaml','w') 
+# for hour in [18]:
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/humppa_080610_'+format(hour,"02d")+'00.txt'
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_afternoon,hour,c4gli_afternoon)
+# file_afternoon.close()
+
+
+
+# path_model = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/HUMPPA/'
+# 
+# file_model    = open(fnout_model+   format(current_station.name,'05d')+'.yaml','w') 
+
+
+records_morning = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='ini',
+                                           refetch_records=True,
+                                           )
+print('records_morning_ldatetime',records_morning.ldatetime)
+
+records_afternoon = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='end',
+                                           refetch_records=True,
+                                           )
+
+# # align afternoon records with noon records, and set same index
+# records_afternoon.index = records_afternoon.ldatetime.dt.date
+# records_afternoon = records_afternoon.loc[records_morning.ldatetime.dt.date]
+# records_afternoon.index = records_morning.index
+# path_exp = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/IOPS_ALL/'
+# 
+# os.system('mkdir -p '+path_exp)
+# file_morning = open(path_soundings+'/'+format(current_station.name,'05d')+'_ini.yaml')
+# file_afternoon = open(path_soundings+'/'+format(current_station.name,'05d')+'_ini.yaml')
+# file_ini = open(path_exp+'/'+format(current_station.name,'05d')+'_ini.yaml','w')
+# file_mod = open(path_exp+'/'+format(current_station.name,'05d')+'_end.yaml','w')
+# 
+# for (STNID,chunk,index),record_morning in records_morning.iterrows():
+#     record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+# 
+#     c4gli_morning = get_record_yaml(file_morning, 
+#                                     record_morning.index_start, 
+#                                     record_morning.index_end,
+#                                     mode='ini')
+#     #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+#     
+#     
+#     c4gli_afternoon = get_record_yaml(file_afternoon, 
+#                                       record_afternoon.index_start, 
+#                                       record_afternoon.index_end,
+#                                     mode='ini')
+# 
+#     c4gli_morning.update(source='pairs',pars={'runtime' : \
+#                         int((c4gli_afternoon.pars.datetime_daylight - 
+#                              c4gli_morning.pars.datetime_daylight).total_seconds())})
+#     c4gli_morning.update(source='manual',
+#                          pars={'sw_ac' : [],'sw_ap': True,'sw_lit': False})
+#     c4gli_morning.dump(file_ini)
+#     
+#     c4gl = class4gl(c4gli_morning)
+#     c4gl.run()
+#     
+#     c4gl.dump(file_mod,\
+#               include_input=False,\
+#               timeseries_only=timeseries_only)
+# file_ini.close()
+# file_mod.close()
+# file_morning.close()
+# file_afternoon.close()
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                            path_exp,\
+#                                            subset='ini',
+#                                            refetch_records=True,
+#                                            )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                            path_exp,\
+#                                            subset='end',
+#                                            refetch_records=True,
+#                                            )
+# 
+# records_mod.index = records_ini.index
+# 
+# # align afternoon records with initial records, and set same index
+# records_afternoon.index = records_afternoon.ldatetime.dt.date
+# records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+# records_afternoon.index = records_ini.index
+
+# stations_for_iter = stations(path_exp)
+# for STNID,station in stations_iterator(stations_for_iter):
+#     records_current_station_index = \
+#             (records_ini.index.get_level_values('STNID') == STNID)
+#     file_current_station_mod = STNID
+# 
+#     with \
+#     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+#     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+#     open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+#         for (STNID,index),record_ini in records_iterator(records_ini):
+#             c4gli_ini = get_record_yaml(file_station_ini, 
+#                                         record_ini.index_start, 
+#                                         record_ini.index_end,
+#                                         mode='ini')
+#             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+# 
+#             record_mod = records_mod.loc[(STNID,index)]
+#             c4gl_mod = get_record_yaml(file_station_mod, 
+#                                         record_mod.index_start, 
+#                                         record_mod.index_end,
+#                                         mode='mod')
+#             record_afternoon = records_afternoon.loc[(STNID,index)]
+#             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+#                                         record_afternoon.index_start, 
+#                                         record_afternoon.index_end,
+#                                         mode='ini')
+
+
+
+# # select the samples of the afternoon list that correspond to the timing of the
+# # morning list
+# records_afternoon = records_afternoon.set_index('ldatetime').loc[records_afternoon.ldatetime)]
+# records_afternoon.index = recods_morning.index
+# 
+# 
+# # create intersectino index
+# index_morning = pd.Index(records_morning.ldatetime.to_date())
+# index_afternoon = pd.Index(records_afternoon.ldatetime.to_date())
+# 
+# for record_morning in records_morning.iterrows():
+#     
+#     c4gl = class4gl(c4gli)
+#     c4gl.run()
+#     c4gl.dump(c4glfile,\
+#               include_input=True,\
+#               timeseries_only=timeseries_only)
+# 
+# # This will cash the observations and model tables per station for
+# # the interface
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=0,\
+#                                    by=2,\
+#                                    subset='ini',
+#                                    refetch_records=True,
+#                                    )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='mod',
+#                                    refetch_records=True,
+#                                    )
+# records_eval = get_records(pd.DataFrame([current_station]),\
+#                                    path_obs,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='eval',
+#                                    refetch_records=True,
+#                                    )
+# 
+# 
+# # mod_scores = pd.DataFrame(index=mod_records.index)
+# # for (STNID,index), current_record_mod in mod_records.iterrows():
+# #     print(STNID,index)
+# #     current_station = STN
+# #     current_record_obs_afternoon = obs_records_afternoon.loc[(STNID,index)]
+# #     current_record_obs = obs_records.loc[(STNID,index)]
+# # 
+# #     record_yaml_mod = get_record_yaml_mod(odirexperiments[keyEXP],\
+# #                                           current_station,\
+# #                                           current_record_mod,\
+# #                                          )
+# # 
+# #     record_yaml_obs = \
+# #             get_record_yaml_obs(odirexperiments[keyEXP],\
+# #                                 current_station,\
+# #                                 current_record_obs,\
+# #                                 suffix='.yaml')
+# # 
+# #     record_yaml_obs_afternoon = \
+# #             get_record_yaml_obs(odir,\
+# #                                 current_station,\
+# #                                 current_record_obs_afternoon,\
+# #                                 suffix='_afternoon.yaml')
+# # 
+# #     hmax = np.max([record_yaml_obs_afternoon.pars.h,\
+# #                    record_yaml_mod.h])
+# #     HEIGHTS = {'h':hmax, '2h':2.*hmax, '3000m':3000.}
+# #     
+# # 
+# #     for height,hvalue in HEIGHTS.items():
+# # 
+# #         lt_obs = (record_yaml_obs_afternoon.air_ap.HAGL < hvalue)
+# #         lt_mod = (record_yaml_mod.air_ap.z < hvalue)
+# #         try:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = \
+# #                 rmse(\
+# #                     record_yaml_obs_afternoon.air_ap.theta[lt_obs],\
+# #                     np.interp(\
+# #                         record_yaml_obs_afternoon.air_ap.HAGL[lt_obs],\
+# #                         record_yaml_mod.air_ap.z[lt_mod],\
+# #                         record_yaml_mod.air_ap.theta[lt_mod]\
+# #                     ))
+# #         except ValueError:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = np.nan
+# #     # # we calculate these things in the interface itself
+# #     # for key in ['q','theta','h']:
+# #     #     mod_records.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_mod.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# #     #     # the actual time of the initial and evaluation sounding can be 
+# #     #     # different, but we consider this as a measurement error for
+# #     #     # the starting and end time of the simulation.
+# #     #     obs_records_afternoon.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_obs_afternoon.pars.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# # mod_scores.to_pickle(odirexperiments[keyEXP]+'/'+format(STNID,'05d')+"_mod_scores.pkl")
+# #         
+# #                 
+# #                 
+# # # for EXP,c4glfile in c4glfiles.items():
+# # #     c4glfile.close()            
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# #     
+# #     # {'Time[min:sec]': None 
+# #     #  'P[hPa]': None, 
+# #     #  'T[C]': None, 
+# #     #  'U[%]': None, 
+# #     #  'Wsp[m/s]': None, 
+# #     #  'Wdir[Grd]': None,
+# #     #  'Lon[°]', 
+# #     #  'Lat[°]', 
+# #     #  'Altitude[m]', 'GeoPot[m']', 'MRI',
+# #     #        'Unnamed: 11', 'RI', 'Unnamed: 13', 'DewPoint[C]', 'Virt. Temp[C]',
+# #     #        'Rs[m/min]D[kg/m3]Azimut[deg]', 'Elevation[deg]', 'Range[m]']
+# #     # }
+# #     # 
+# #     # #pivotrows =
+# #     # #{
+# # 
+# # 
+# # 
diff --git a/class4gl/setup/setup_humppa_noon.py b/class4gl/setup/setup_humppa_noon.py
new file mode 100644
index 0000000..72bbf89
--- /dev/null
+++ b/class4gl/setup/setup_humppa_noon.py
@@ -0,0 +1,733 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import Pysolar
+import sys
+import pytz
+sys.path.insert(0,'/user/home/gent/vsc422/vsc42247/software/class4gl/class4gl/')
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+
+globaldata = data_global()
+globaldata.load_datasets(recalc=0)
+
+Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+epsilon = Rd/Rv # or mv/md
+
+
+def replace_iter(iterable, search, replace):
+    for value in iterable:
+        value.replace(search, replace)
+        yield value
+
+from class4gl import blh,class4gl_input
+
+# definition of the humpa station
+current_station = pd.Series({ "latitude"  : 61.8448,
+                  "longitude" : 24.2882,
+                  "name" : "the HUMMPA experiment"
+                })
+current_station.name = 90000
+
+# we define the columns ourselves because it is a mess in the file itself.
+columns =\
+['Time[min:sec]',
+ 'P[hPa]',
+ 'T[C]',
+ 'U[%]',
+ 'Wsp[m/s]',
+ 'Wdir[Grd]',
+ 'Lon[°]',
+ 'Lat[°]',
+ 'Altitude[m]',
+ 'GeoPot[m]',
+ 'MRI',
+ 'RI',    
+ 'DewPoint[C]',
+ 'Virt. Temp[C]',
+ 'Rs[m/min]',
+ 'D[kg/m3]',
+ 'Azimut[°]',
+ 'Elevation[°]',
+ 'Range[m]',
+]
+
+
+HOUR_FILES = \
+{ 
+  # dt.datetime(2010,7,12,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071210_0300.txt'],'afternoon':[15,'humppa_071210_1500.txt']},
+  dt.datetime(2010,7,13,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071310_0300.txt'],'afternoon':[12,'humppa_071310_1200.txt']},
+#  dt.datetime(2010,7,14,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071410_0300.txt'],'afternoon':[16,'humppa_071410_1600.txt']},
+  #dt.datetime(2010,7,15,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071510_0300.txt'],'afternoon':[15,'humppa_071510_1500.txt']},
+  #dt.datetime(2010,7,16,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071610_0300.txt'],'afternoon':[21,'humppa_071610_2100.txt']},
+  #dt.datetime(2010,7,17,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071710_0300.txt'],'afternoon':[18,'humppa_071710_1800.txt']},
+  #dt.datetime(2010,7,18,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071810_0300.txt'],'afternoon':[21,'humppa_071810_2100.txt']},
+  dt.datetime(2010,7,19,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_071910_0300.txt'],'afternoon':[9,'humppa_071910_0900.txt']},
+#  dt.datetime(2010,7,20):{'morning':[4,'humppa_072010_0400.txt'],'afternoon':[15,'humppa_072010_1500.txt']},
+#  dt.datetime(2010,7,21,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072110_0300.txt'],'afternoon':[21,'humppa_072110_2100.txt']},
+  dt.datetime(2010,7,22,0,0,0,0,pytz.UTC):{'morning':[4,'humppa_072210_0400.txt'],'afternoon':[12,'humppa_072210_1200.txt']},
+ # something is wrong with ths profile
+ # dt.datetime(2010,7,23,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072310_0300.txt'],'afternoon':[15,'humppa_072310_1500.txt']},
+#  dt.datetime(2010,7,24,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072410_0300.txt'],'afternoon':[16,'humppa_072410_1600.txt']},
+#  dt.datetime(2010,7,25,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072510_0300.txt'],'afternoon':[21,'humppa_072510_2100.txt']},
+#  dt.datetime(2010,7,26,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072610_0300.txt'],'afternoon':[21,'humppa_072610_2100.txt']},
+#  dt.datetime(2010,7,27,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072710_0300.txt'],'afternoon':[15,'humppa_072710_1500.txt']},
+#  dt.datetime(2010,7,28,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_072810_0300.txt'],'afternoon':[15,'humppa_072810_1500.txt']},
+  dt.datetime(2010,7,29,0,0,0,0,pytz.UTC):{'morning':[4,'humppa_072910_0400.txt'],'afternoon':[12,'humppa_072910_1200.txt']},
+#  dt.datetime(2010,7,30,0,0,0,0,pytz.UTC):{'morning':[9,'humppa_073010_0900.txt'],'afternoon':[15,'humppa_073010_1500.txt']},
+  dt.datetime(2010,7,31,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_073110_0300_01.txt'],'afternoon':[12,'humppa_073110_1200.txt']},
+#  dt.datetime(2010,8, 1,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080110_0300.txt'],'afternoon':[18,'humppa_080110_1800.txt']},
+  dt.datetime(2010,8, 2,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080210_0300.txt'],'afternoon':[9,'humppa_080210_0900.txt']},
+#  dt.datetime(2010,8, 3,0,0,0,0,pytz.UTC):{'morning':[9,'humppa_080310_0900.txt'],'afternoon':[18,'humppa_080310_1800.txt']},
+#  dt.datetime(2010,8, 3,0,0,0,0,pytz.UTC):{'morning':[8,'humppa_080410_0800.txt'],'afternoon':[18,'humppa_080410_1800.txt']},
+#  dt.datetime(2010,8, 5,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080510_0300.txt'],'afternoon':[18,'humppa_080510_1800.txt']},
+#  dt.datetime(2010,8, 6,0,0,0,0,pytz.UTC):{'morning':[4,'humppa_080610_0400.txt'],'afternoon':[18,'humppa_080610_1800.txt']},
+#  dt.datetime(2010,8, 7,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080710_0300.txt'],'afternoon':[18,'humppa_080710_1800.txt']},
+#  dt.datetime(2010,8, 8,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_080810_0300.txt'],'afternoon':[18,'humppa_080810_1800.txt']},
+  dt.datetime(2010,8,10,0,0,0,0,pytz.UTC):{'morning':[3,'humppa_081010_0300.txt'],'afternoon':[12,'humppa_081010_1200.txt']},
+}
+
+
+
+
+
+
+#only include the following timeseries in the model output
+timeseries_only = \
+['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+ 'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+ 'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+ 'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+ 'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+def humppa_parser(balloon_file,file_sounding,ldate,hour,c4gli=None):
+        #balloon_conv = replace_iter(balloon_file,"°","deg")
+        #readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()]
+        #air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15)
+        air_balloon_in = pd.read_fwf(balloon_file,
+                                     widths=[14]*19,
+                                     skiprows=9,
+                                     skipfooter=15,
+                                     decimal=',',
+                                     header=None,
+                                     names = columns,
+                                     na_values='-----')
+    
+        rowmatches = {
+            't':      lambda x: x['T[C]']+273.15,
+            #'tv':     lambda x: x['Virt. Temp[C]']+273.15,
+            'p':      lambda x: x['P[hPa]']*100.,
+            'u':      lambda x: x['Wsp[m/s]'] * np.sin((90.-x['Wdir[Grd]'])/180.*np.pi),
+            'v':      lambda x: x['Wsp[m/s]'] * np.cos((90.-x['Wdir[Grd]'])/180.*np.pi),
+            'z':      lambda x: x['Altitude[m]'],
+            'q':      lambda x: np.clip((1. - (273.15+x['Virt. Temp[C]'])/(273.15+x['T[C]']))/(1. - 1./epsilon),a_min=0.,a_max=None),
+        }
+        
+        air_balloon = pd.DataFrame()
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon_in)
+        
+        rowmatches = {
+            'R' :    lambda x: (Rd*(1.-x.q) + Rv*x.q),
+            'theta': lambda x: (x['t']) * (x['p'][0]/x['p'])**(x['R']/cp),
+            'thetav': lambda x: x.theta  + 0.61 * x.theta * x.q
+        }
+        
+        for varname,lfunction in rowmatches.items():
+            air_balloon[varname] = lfunction(air_balloon)
+        
+        dpars = {}
+        dpars['longitude']  = current_station['longitude']
+        dpars['latitude']  = current_station['latitude'] 
+        
+        dpars['STNID'] = current_station.name
+        
+
+        # there are issues with the lower measurements in the HUMPPA campaign,
+        # for which a steady decrease of potential temperature is found, which
+        # is unrealistic.  Here I filter them away
+        ifirst = 0
+        while  (air_balloon.theta.iloc[ifirst+1] < air_balloon.theta.iloc[ifirst]):
+            ifirst = ifirst+1
+        print ('ifirst:',ifirst)
+        air_balloon = air_balloon.iloc[ifirst:].reset_index().drop(['index'],axis=1)
+        
+        is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)
+        valid_indices = air_balloon.index[is_valid].values
+        
+        air_ap_mode='b'
+        
+        if len(valid_indices) > 0:
+            dpars['h'],dpars['h_u'],dpars['h_l'] =\
+                blh(air_balloon.z,air_balloon.thetav,air_balloon_in['Wsp[m/s]'])
+            dpars['h_b'] = np.max((dpars['h'],10.))
+            dpars['h_u'] = np.max((dpars['h_u'],10.)) #upper limit of mixed layer height
+            dpars['h_l'] = np.max((dpars['h_l'],10.)) #low limit of mixed layer height
+            dpars['h_e'] = np.abs( dpars['h_u'] - dpars['h_l']) # error of mixed-layer height
+            dpars['h'] = np.round(dpars['h_'+air_ap_mode],1)
+        else:
+            dpars['h_u'] =np.nan
+            dpars['h_l'] =np.nan
+            dpars['h_e'] =np.nan
+            dpars['h'] =np.nan
+        
+        
+        
+        if ~np.isnan(dpars['h']):
+            dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]]
+        else:
+            dpars['Ps'] = np.nan
+        
+        if ~np.isnan(dpars['h']):
+        
+            # determine mixed-layer properties (moisture, potential temperature...) from profile
+            
+            # ... and those of the mixed layer
+            is_valid_below_h = is_valid & (air_balloon.z < dpars['h'])
+            valid_indices_below_h =  air_balloon.index[is_valid_below_h].values
+            if len(valid_indices) > 1:
+                if len(valid_indices_below_h) >= 3.:
+                    ml_mean = air_balloon[is_valid_below_h].mean()
+                else:
+                    ml_mean = air_balloon.iloc[valid_indices[0]:valid_indices[1]].mean()
+            elif len(valid_indices) == 1:
+                ml_mean = (air_balloon.iloc[0:1]).mean()
+            else:
+                temp =  pd.DataFrame(air_balloon)
+                temp.iloc[0] = np.nan
+                ml_mean = temp
+                       
+            dpars['theta']= ml_mean.theta
+            dpars['q']    = ml_mean.q
+            dpars['u']    = ml_mean.u
+            dpars['v']    = ml_mean.v 
+        else:
+            dpars['theta'] = np.nan
+            dpars['q'] = np.nan
+            dpars['u'] = np.nan
+            dpars['v'] = np.nan
+        
+        air_ap_head = air_balloon[0:0] #pd.DataFrame(columns = air_balloon.columns)
+        # All other  data points above the mixed-layer fit
+        air_ap_tail = air_balloon[air_balloon.z > dpars['h']]
+
+
+
+        air_ap_head.z = pd.Series(np.array([2.,dpars['h'],dpars['h']]))
+        jump = air_ap_head.iloc[0] * np.nan
+        
+        if air_ap_tail.shape[0] > 1:
+        
+            # we originally used THTA, but that has another definition than the
+            # variable theta that we need which should be the temperature that
+            # one would have if brought to surface (NOT reference) pressure.
+            for column in ['theta','q','u','v']:
+               
+               # initialize the profile head with the mixed-layer values
+               air_ap_head[column] = ml_mean[column]
+               # calculate jump values at mixed-layer height, which will be
+               # added to the third datapoint of the profile head
+               jump[column] = (air_ap_tail[column].iloc[1]\
+                               -\
+                               air_ap_tail[column].iloc[0])\
+                              /\
+                              (air_ap_tail.z.iloc[1]\
+                               - air_ap_tail.z.iloc[0])\
+                              *\
+                              (dpars['h']- air_ap_tail.z.iloc[0])\
+                              +\
+                              air_ap_tail[column].iloc[0]\
+                              -\
+                              ml_mean[column] 
+               if column == 'theta':
+                  # for potential temperature, we need to set a lower limit to
+                  # avoid the model to crash
+                  jump.theta = np.max((0.1,jump.theta))
+        
+               air_ap_head[column][2] += jump[column]
+        
+        air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 +air_ap_head.v**2)
+
+
+
+        # only select samples monotonically increasing with height
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        for ibottom in range(1,len(air_ap_tail_orig)):
+            if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z +10.:
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],ignore_index=True)
+
+        # make theta increase strong enough to avoid numerical
+        # instability
+        air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        air_ap_tail = pd.DataFrame()
+        #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        theta_low = air_ap_head['theta'].iloc[2]
+        z_low = air_ap_head['z'].iloc[2]
+        ibottom = 0
+        for itop in range(0,len(air_ap_tail_orig)):
+            theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+            z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+            if (
+                #(z_mean > z_low) and \
+                (z_mean > (z_low+10.)) and \
+                #(theta_mean > (theta_low+0.2) ) and \
+                #(theta_mean > (theta_low+0.2) ) and \
+                 (((theta_mean - theta_low)/(z_mean - z_low)) > 0.0001)):
+
+                air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+                ibottom = itop+1
+                theta_low = air_ap_tail.theta.iloc[-1]
+                z_low =     air_ap_tail.z.iloc[-1]
+            # elif  (itop > len(air_ap_tail_orig)-10):
+            #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        
+        air_ap = \
+            pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+
+        # # make theta increase strong enough to avoid numerical
+        # # instability
+        # air_ap_tail_orig = pd.DataFrame(air_ap_tail)
+        # air_ap_tail = pd.DataFrame()
+        # #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
+        # theta_low = air_ap_head['theta'].iloc[2]
+        # z_low = air_ap_head['z'].iloc[2]
+        # ibottom = 0
+        # for itop in range(0,len(air_ap_tail_orig)):
+        #     theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
+        #     z_mean =     air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
+        #     if ((theta_mean > (theta_low+0.2) ) and \
+        #          (((theta_mean - theta_low)/(z_mean - z_low)) > 0.001)):
+
+        #         air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
+        #         ibottom = itop+1
+        #         theta_low = air_ap_tail.theta.iloc[-1]
+        #         z_low =     air_ap_tail.z.iloc[-1]
+        #     # elif  (itop > len(air_ap_tail_orig)-10):
+        #     #     air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
+        # 
+        # air_ap = \
+        #     pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
+        
+        # we copy the pressure at ground level from balloon sounding. The
+        # pressure at mixed-layer height will be determined internally by class
+        
+        rho        = 1.2                   # density of air [kg m-3]
+        g          = 9.81                  # gravity acceleration [m s-2]
+        
+        air_ap['p'].iloc[0] =dpars['Ps'] 
+        air_ap['p'].iloc[1] =(dpars['Ps'] - rho * g * dpars['h'])
+        air_ap['p'].iloc[2] =(dpars['Ps'] - rho * g * dpars['h'] -0.1)
+        
+        
+        dpars['lat'] = dpars['latitude']
+        # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich)
+        dpars['lon'] = 0.
+        # this is the real longitude that will be used to extract ground data
+        
+        dpars['ldatetime'] = ldate+dt.timedelta(hours=hour)
+        dpars['datetime'] =  dpars['ldatetime'] + dt.timedelta(hours=-3)
+        dpars['doy'] = dpars['datetime'].timetuple().tm_yday
+        
+        dpars['SolarAltitude'] = \
+                                Pysolar.GetAltitude(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        dpars['SolarAzimuth'] =  Pysolar.GetAzimuth(\
+                                    dpars['latitude'],\
+                                    dpars['longitude'],\
+                                    dpars['datetime']\
+                                )
+        
+        
+        dpars['lSunrise'], dpars['lSunset'] \
+        =  Pysolar.util.GetSunriseSunset(dpars['latitude'],
+                                         0.,
+                                         dpars['ldatetime'],0.)
+        
+        # Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually write local solar time, we need to assign the timezone to UTC (which is WRONG!!!). Otherwise ruby cannot understand it (it always converts tolocal computer time :( ). 
+        dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise'])
+        dpars['lSunset'] = pytz.utc.localize(dpars['lSunset'])
+        
+        # This is the nearest datetime when the sun is up (for class)
+        dpars['ldatetime_daylight'] = \
+                                np.min(\
+                                    (np.max(\
+                                        (dpars['ldatetime'],\
+                                         dpars['lSunrise'])\
+                                     ),\
+                                     dpars['lSunset']\
+                                    )\
+                                )
+        # apply the same time shift for UTC datetime
+        dpars['datetime_daylight'] = dpars['datetime'] \
+                                    +\
+                                    (dpars['ldatetime_daylight']\
+                                     -\
+                                     dpars['ldatetime'])
+        
+        
+        # We set the starting time to the local sun time, since the model 
+        # thinks we are always at the meridian (lon=0). This way the solar
+        # radiation is calculated correctly.
+        dpars['tstart'] = dpars['ldatetime_daylight'].hour \
+                         + \
+                         dpars['ldatetime_daylight'].minute/60.\
+                         + \
+                         dpars['ldatetime_daylight'].second/3600.
+        
+        dpars['sw_lit'] = False
+        # convert numpy types to native python data types. This provides
+        # cleaner data IO with yaml:
+        for key,value in dpars.items():
+            if type(value).__module__ == 'numpy':
+                dpars[key] = dpars[key].item()
+        
+                decimals = {'p':0,'t':2,'theta':4, 'z':2, 'q':5, 'u':4, 'v':4}
+        # 
+                for column,decimal in decimals.items():
+                    air_balloon[column] = air_balloon[column].round(decimal)
+                    air_ap[column] = air_ap[column].round(decimal)
+        
+        updateglobal = False
+        if c4gli is None:
+            c4gli = class4gl_input()
+            updateglobal = True
+        
+        print('updating...')
+        print(column)
+        c4gli.update(source='humppa',\
+                    # pars=pars,
+                    pars=dpars,\
+                    air_balloon=air_balloon,\
+                    air_ap=air_ap)
+        if updateglobal:
+            c4gli.get_global_input(globaldata)
+
+        # if profile_ini:
+        #     c4gli.runtime = 10 * 3600
+
+        c4gli.dump(file_sounding)
+        
+        # if profile_ini:
+        #     c4gl = class4gl(c4gli)
+        #     c4gl.run()
+        #     c4gl.dump(file_model,\
+        #               include_input=True,\
+        #               timeseries_only=timeseries_only)
+        #     
+        #     # This will cash the observations and model tables per station for
+        #     # the interface
+        # 
+        # if profile_ini:
+        #     profile_ini=False
+        # else:
+        #     profile_ini=True
+        return c4gli
+
+
+path_soundings = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/IOPS_NOON/'
+
+
+file_morning = open(path_soundings+format(current_station.name,'05d')+'_morning.yaml','w') 
+for date,pair  in HOUR_FILES.items(): 
+    print(pair['morning'])
+    humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/'+pair['morning'][1]
+    print(humpafn)
+    balloon_file = open(humpafn,'r',encoding='latin-1')
+
+    c4gli_morning = humppa_parser(balloon_file,file_morning,date,pair['morning'][0])
+    print('c4gli_morning_ldatetime 0',c4gli_morning.pars.ldatetime)
+file_morning.close()
+
+file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_afternoon.yaml','w') 
+for date,pair  in HOUR_FILES.items(): 
+    humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/'+pair['afternoon'][1]
+    balloon_file = open(humpafn,'r',encoding='latin-1')
+
+    c4gli_afternoon = humppa_parser(balloon_file,file_afternoon,date,pair['afternoon'][0])
+    print('c4gli_afternoon_ldatetime 0',c4gli_afternoon.pars.ldatetime)
+file_afternoon.close()
+ 
+
+# file_morning = open(path_soundings+format(current_station.name,'05d')+'_morning.yaml','w') 
+# for date,pair  in HOUR_FILES.items(): 
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/'+pair['morning'][1],
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_morning,hour,c4gli_morning)
+#     print('c4gli_morning_ldatetime 1',c4gli_morning.pars.ldatetime)
+# file_morning.close()
+# 
+# file_afternoon = open(path_soundings+format(current_station.name,'05d')+'_afternoon.yaml','w') 
+# for hour in [18]:
+#     humpafn ='/kyukon/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/HUMPPA/humppa_080610_'+format(hour,"02d")+'00.txt'
+#     balloon_file = open(humpafn,'r',encoding='latin-1')
+# 
+#     humppa_parser(balloon_file,file_afternoon,hour,c4gli_afternoon)
+# file_afternoon.close()
+
+
+
+# path_model = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/HUMPPA/'
+# 
+# file_model    = open(fnout_model+   format(current_station.name,'05d')+'.yaml','w') 
+
+
+records_morning = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='morning',
+                                           refetch_records=True,
+                                           )
+print('records_morning_ldatetime',records_morning.ldatetime)
+
+records_afternoon = get_records(pd.DataFrame([current_station]),\
+                                           path_soundings,\
+                                           subset='afternoon',
+                                           refetch_records=True,
+                                           )
+
+# align afternoon records with noon records, and set same index
+records_afternoon.index = records_afternoon.ldatetime.dt.date
+records_afternoon = records_afternoon.loc[records_morning.ldatetime.dt.date]
+records_afternoon.index = records_morning.index
+path_exp = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/IOPS_NOON/'
+
+os.system('mkdir -p '+path_exp)
+file_morning = open(path_soundings+'/'+format(current_station.name,'05d')+'_morning.yaml')
+file_afternoon = open(path_soundings+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+file_ini = open(path_exp+'/'+format(current_station.name,'05d')+'_ini.yaml','w')
+file_mod = open(path_exp+'/'+format(current_station.name,'05d')+'_mod.yaml','w')
+
+for (STNID,chunk,index),record_morning in records_morning.iterrows():
+    record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+
+    c4gli_morning = get_record_yaml(file_morning, 
+                                    record_morning.index_start, 
+                                    record_morning.index_end,
+                                    mode='ini')
+    #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+    
+    
+    c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                      record_afternoon.index_start, 
+                                      record_afternoon.index_end,
+                                    mode='ini')
+
+    c4gli_morning.update(source='pairs',pars={'runtime' : \
+                        int((c4gli_afternoon.pars.datetime_daylight - 
+                             c4gli_morning.pars.datetime_daylight).total_seconds())})
+    c4gli_morning.update(source='manual',
+                         pars={'sw_ac' : [],'sw_ap': True,'sw_lit': False})
+    c4gli_morning.dump(file_ini)
+    
+    c4gl = class4gl(c4gli_morning)
+    c4gl.run()
+    
+    c4gl.dump(file_mod,\
+              include_input=False,\
+              timeseries_only=timeseries_only)
+file_ini.close()
+file_mod.close()
+file_morning.close()
+file_afternoon.close()
+
+records_ini = get_records(pd.DataFrame([current_station]),\
+                                           path_exp,\
+                                           subset='ini',
+                                           refetch_records=True,
+                                           )
+records_mod = get_records(pd.DataFrame([current_station]),\
+                                           path_exp,\
+                                           subset='mod',
+                                           refetch_records=True,
+                                           )
+
+records_mod.index = records_ini.index
+
+# align afternoon records with initial records, and set same index
+records_afternoon.index = records_afternoon.ldatetime.dt.date
+records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+records_afternoon.index = records_ini.index
+
+# stations_for_iter = stations(path_exp)
+# for STNID,station in stations_iterator(stations_for_iter):
+#     records_current_station_index = \
+#             (records_ini.index.get_level_values('STNID') == STNID)
+#     file_current_station_mod = STNID
+# 
+#     with \
+#     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+#     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+#     open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+#         for (STNID,index),record_ini in records_iterator(records_ini):
+#             c4gli_ini = get_record_yaml(file_station_ini, 
+#                                         record_ini.index_start, 
+#                                         record_ini.index_end,
+#                                         mode='ini')
+#             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+# 
+#             record_mod = records_mod.loc[(STNID,index)]
+#             c4gl_mod = get_record_yaml(file_station_mod, 
+#                                         record_mod.index_start, 
+#                                         record_mod.index_end,
+#                                         mode='mod')
+#             record_afternoon = records_afternoon.loc[(STNID,index)]
+#             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+#                                         record_afternoon.index_start, 
+#                                         record_afternoon.index_end,
+#                                         mode='ini')
+
+
+
+# # select the samples of the afternoon list that correspond to the timing of the
+# # morning list
+# records_afternoon = records_afternoon.set_index('ldatetime').loc[records_afternoon.ldatetime)]
+# records_afternoon.index = recods_morning.index
+# 
+# 
+# # create intersectino index
+# index_morning = pd.Index(records_morning.ldatetime.to_date())
+# index_afternoon = pd.Index(records_afternoon.ldatetime.to_date())
+# 
+# for record_morning in records_morning.iterrows():
+#     
+#     c4gl = class4gl(c4gli)
+#     c4gl.run()
+#     c4gl.dump(c4glfile,\
+#               include_input=True,\
+#               timeseries_only=timeseries_only)
+# 
+# # This will cash the observations and model tables per station for
+# # the interface
+# 
+# records_ini = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=0,\
+#                                    by=2,\
+#                                    subset='ini',
+#                                    refetch_records=True,
+#                                    )
+# records_mod = get_records(pd.DataFrame([current_station]),\
+#                                    path_mod,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='mod',
+#                                    refetch_records=True,
+#                                    )
+# records_eval = get_records(pd.DataFrame([current_station]),\
+#                                    path_obs,\
+#                                    start=1,\
+#                                    by=2,\
+#                                    subset='eval',
+#                                    refetch_records=True,
+#                                    )
+# 
+# 
+# # mod_scores = pd.DataFrame(index=mod_records.index)
+# # for (STNID,index), current_record_mod in mod_records.iterrows():
+# #     print(STNID,index)
+# #     current_station = STN
+# #     current_record_obs_afternoon = obs_records_afternoon.loc[(STNID,index)]
+# #     current_record_obs = obs_records.loc[(STNID,index)]
+# # 
+# #     record_yaml_mod = get_record_yaml_mod(odirexperiments[keyEXP],\
+# #                                           current_station,\
+# #                                           current_record_mod,\
+# #                                          )
+# # 
+# #     record_yaml_obs = \
+# #             get_record_yaml_obs(odirexperiments[keyEXP],\
+# #                                 current_station,\
+# #                                 current_record_obs,\
+# #                                 suffix='.yaml')
+# # 
+# #     record_yaml_obs_afternoon = \
+# #             get_record_yaml_obs(odir,\
+# #                                 current_station,\
+# #                                 current_record_obs_afternoon,\
+# #                                 suffix='_afternoon.yaml')
+# # 
+# #     hmax = np.max([record_yaml_obs_afternoon.pars.h,\
+# #                    record_yaml_mod.h])
+# #     HEIGHTS = {'h':hmax, '2h':2.*hmax, '3000m':3000.}
+# #     
+# # 
+# #     for height,hvalue in HEIGHTS.items():
+# # 
+# #         lt_obs = (record_yaml_obs_afternoon.air_ap.HAGL < hvalue)
+# #         lt_mod = (record_yaml_mod.air_ap.z < hvalue)
+# #         try:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = \
+# #                 rmse(\
+# #                     record_yaml_obs_afternoon.air_ap.theta[lt_obs],\
+# #                     np.interp(\
+# #                         record_yaml_obs_afternoon.air_ap.HAGL[lt_obs],\
+# #                         record_yaml_mod.air_ap.z[lt_mod],\
+# #                         record_yaml_mod.air_ap.theta[lt_mod]\
+# #                     ))
+# #         except ValueError:
+# #             mod_scores.at[(STNID,index),'rmse_'+height] = np.nan
+# #     # # we calculate these things in the interface itself
+# #     # for key in ['q','theta','h']:
+# #     #     mod_records.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_mod.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# #     #     # the actual time of the initial and evaluation sounding can be 
+# #     #     # different, but we consider this as a measurement error for
+# #     #     # the starting and end time of the simulation.
+# #     #     obs_records_afternoon.at[(STNID,index),'d'+key+'dt'] = \
+# #     #                 (record_yaml_obs.pars.__dict__[key] -  \
+# #     #                  record_yaml_obs_afternoon.pars.__dict__[key]\
+# #     #                 )/(record_yaml_obs_afternoon.pars.ldatetime - \
+# #     #                    record_yaml_obs.pars.ldatetime).total_seconds()
+# # 
+# # mod_scores.to_pickle(odirexperiments[keyEXP]+'/'+format(STNID,'05d')+"_mod_scores.pkl")
+# #         
+# #                 
+# #                 
+# # # for EXP,c4glfile in c4glfiles.items():
+# # #     c4glfile.close()            
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# # 
+# #     
+# #     # {'Time[min:sec]': None 
+# #     #  'P[hPa]': None, 
+# #     #  'T[C]': None, 
+# #     #  'U[%]': None, 
+# #     #  'Wsp[m/s]': None, 
+# #     #  'Wdir[Grd]': None,
+# #     #  'Lon[°]', 
+# #     #  'Lat[°]', 
+# #     #  'Altitude[m]', 'GeoPot[m']', 'MRI',
+# #     #        'Unnamed: 11', 'RI', 'Unnamed: 13', 'DewPoint[C]', 'Virt. Temp[C]',
+# #     #        'Rs[m/min]D[kg/m3]Azimut[deg]', 'Elevation[deg]', 'Range[m]']
+# #     # }
+# #     # 
+# #     # #pivotrows =
+# #     # #{
+# # 
+# # 
+# # 
diff --git a/class4gl/setup/setup_igra.py b/class4gl/setup/setup_igra.py
new file mode 100644
index 0000000..f8e72f4
--- /dev/null
+++ b/class4gl/setup/setup_igra.py
@@ -0,0 +1,430 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thursday, March 29, 11:30 AM
+
+@author: Hendrik Wouters
+
+The dry-2-dry global radio sounding experiment.
+
+usage:
+    python setup_global.py 
+    where  is an integer indicating the row index of the station list
+    under args.path_output+'/'+fn_stations (see below)
+
+this scripts should be called from the pbs script setup_global.pbs
+
+
+
+dependencies:
+    - pandas
+    - class4gl
+    - data_soundings
+
+
+"""
+
+""" import libraries """
+import pandas as pd
+import sys
+#import copy as cp
+import numpy as np
+#from sklearn.metrics import mean_squared_error
+import logging
+import datetime as dt
+import os
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_input')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+# parser.add_argument('--first_YYYYMMDD',default="19810101")
+# parser.add_argument('--last_YYYYMMDD',default="20180101")
+parser.add_argument('--startyear',default="1981")
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--check_inputdata',default='True') # run a specific station id
+parser.add_argument('--force_pars',default='') # run a specific station id
+# parser.add_argument('--error_handling',default='dump_on_success')
+# parser.add_argument('--subset_output',default='morning') # this tells which yaml subset
+
+
+# args.path_output = "/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/GLOBAL/"
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+fn_stations = args.path_input+'/igra-stations.txt'
+
+def isfloat(value):
+  try:
+    float(value)
+    return True
+  except ValueError:
+    return False
+
+
+#calculate the root mean square error
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+from class4gl import class4gl_input, data_global,class4gl
+from data_soundings import wyoming
+#from data_global import data_global
+
+# iniitialize global data
+globaldata = data_global()
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+# read the list of stations with valid ground data (list generated with
+# get_valid_stations.py)
+# args.path_input = "/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/"
+
+# df_stations = pd.read_fwf(fn_stations,names=['Country code',\
+#                                                'ID',\
+#                                                'Name',\
+#                                                'latitude',\
+#                                                'longitude',\
+#                                                'height',\
+#                                                'unknown',\
+#                                                'startyear',\
+#                                                'endyear'])
+# 
+
+# ===============================
+print("getting a list of stations")
+# ===============================
+all_stations = stations(args.path_input,refetch_stations=False)
+df_stations = all_stations.table
+df_stations.columns
+
+if args.station_id is not None:
+    df_stations = df_stations.query('STNID == '+args.station_id)
+else:
+    if args.last_station_row is not None:
+        df_stations = df_stations[:(int(args.last_station_row)+1)]
+    if args.first_station_row is not None:
+        df_stations = df_stations[int(args.first_station_row):]
+
+STNlist = list(df_stations.iterrows())
+
+os.system('mkdir -p '+args.path_output)
+for iSTN,STN in STNlist:  
+    one_run = False
+# for iSTN,STN in STNlist[5:]:  
+    
+    fnout = args.path_output+"/"+format(STN.name,'05d')+"_ini.yaml"
+    fnout_afternoon = args.path_output+"/"+format(STN.name,'05d')+"_end.yaml"
+    
+    fnout_diag = args.path_output+"/"+format(STN.name,'05d')+"_diag.pkl"
+
+    # c4glfiles = dict([(EXP,odirexperiments[EXP]+'/'+format(STN['ID'],'05d')+'.yaml') \
+    #                   for EXP in experiments.keys()])
+        
+    dict_diag_station = {}
+    df_logic_morning = pd.DataFrame()
+    df_logic_afternoon = pd.DataFrame()
+    with open(fnout,'w') as fileout, \
+         open(fnout_afternoon,'w') as fileout_afternoon:
+        wy_strm = wyoming(PATH=args.path_input, STNM=STN.name)
+        wy_strm.set_STNM(int(STN.name))
+
+        # we consider all soundings from 1981 onwards
+        wy_strm.find_first(year=int(args.startyear))
+        #wy_strm.find(dt.datetime(2004,10,19,6))
+        
+        c4gli = class4gl_input(debug_level=logging.INFO)
+        c4gli_afternoon = class4gl_input(debug_level=logging.INFO)
+        # so we continue as long as we can find a new sounding
+                
+        while wy_strm.current is not None:
+            
+            c4gli.clear()
+            c4gli.get_profile_wyoming(wy_strm)
+            #import pdb;pdb.set_trace()
+            #print(STN['ID'],c4gli.pars.datetime)
+            #c4gli.get_global_input(globaldata)
+
+            print(c4gli.pars.STNID, c4gli.pars.ldatetime)
+
+            logic = dict()
+            logic['morning'] =  (c4gli.pars.ldatetime.hour <= 12.)
+
+            # Sounding should have taken place after 3 hours before sunrise.
+            # Note that the actual simulation only start at sunrise
+            # (specified by ldatetime_daylight), so the ABL cooling af the time
+            # before sunrise is ignored by the simulation.
+            logic['daylight'] = \
+                ((c4gli.pars.ldatetime - 
+                  c4gli.pars.lSunrise).total_seconds()/3600. >= -3)
+            
+            logic['springsummer'] = (c4gli.pars.theta > 278.)
+            print(c4gli.pars.theta)
+            
+            # we take 3000 because previous analysis (ie., HUMPPA) has
+            # focussed towards such altitude
+            le3000 = (c4gli.air_balloon.z <= 3000.)
+            logic['7measurements'] = (np.sum(le3000) >= 7) 
+
+            leh = (c4gli.air_balloon.z < c4gli.pars.h)
+
+            logic['mlerrlow'] = (\
+                    (len(np.where(leh)[0]) > 0) and \
+                    # in cases where humidity is not defined, the mixed-layer
+                    # values get corr
+                    (not np.isnan(c4gli.pars.theta)) and (
+                    # in cases where humidity is not defined, the mixed-layer
+                    (rmse(c4gli.air_balloon.theta[leh] , c4gli.pars.theta,filternan_actual=True) < 1.5))\
+                          )
+    
+
+            logic['mlherrlow'] = (c4gli.pars.h_e <= 150.) ###350
+            
+            print('logic:', logic)
+            # the result
+            morning_ok = np.mean(list(logic.values()))
+            logic['morning_ok'] = morning_ok
+            print(morning_ok,c4gli.pars.ldatetime)
+
+
+
+            # except:
+            #     morning_ok =False
+            #     print('obtain morning not good')
+
+            # the next sounding will be used either for an afternoon sounding
+            # or for the morning sounding of the next day.
+            wy_strm.find_next()
+            # If the morning is ok, then we try to find a decent afternoon
+            # sounding
+            logic_afternoon_def =dict()
+            df_logic_morning = df_logic_morning.append(logic,ignore_index=True)
+            if morning_ok == 1.:
+                print('MORNING OK!')
+                # we get the current date
+                current_date = dt.date(c4gli.pars.ldatetime.year, \
+                                       c4gli.pars.ldatetime.month, \
+                                       c4gli.pars.ldatetime.day)
+                c4gli_afternoon.clear()
+                print('AFTERNOON PROFILE CLEARED')
+                try:
+                    c4gli_afternoon.get_profile_wyoming(wy_strm)
+                    print('AFTERNOON PROFILE OK')
+
+                    if wy_strm.current is not None:
+                        current_date_afternoon = \
+                                   dt.date(c4gli_afternoon.pars.ldatetime.year, \
+                                           c4gli_afternoon.pars.ldatetime.month, \
+                                           c4gli_afternoon.pars.ldatetime.day)
+                    else:
+                        # a dummy date: this will be ignored anyway
+                        current_date_afternoon = dt.date(1900,1,1)
+
+                    # we will dump the latest afternoon sounding that fits the
+                    # minimum criteria specified by logic_afternoon
+                    print(current_date,current_date_afternoon)
+                    c4gli_afternoon_for_dump = None
+                    afternoon_first = True
+                    while ((current_date_afternoon == current_date) and \
+                           (wy_strm.current is not None)):
+                        logic_afternoon =dict()
+
+                        logic_afternoon['afternoon'] = \
+                            (c4gli_afternoon.pars.ldatetime.hour >= 12.)
+                        # the sounding should have taken place before 1 hours
+                        # before sunset. This is to minimize the chance that a
+                        # stable boundary layer (yielding very low mixed layer
+                        # heights) is formed which can not be represented by
+                        # class.
+                        logic_afternoon['afternoon_daylight'] = \
+                          ((c4gli_afternoon.pars.ldatetime - \
+                            c4gli_afternoon.pars.lSunset \
+                           ).total_seconds()/3600. <= -1.)
+
+                        le3000_afternoon = \
+                            (c4gli_afternoon.air_balloon.z <= 3000.)
+                        logic_afternoon['afternoon_7measurements'] = \
+                            (np.sum(le3000_afternoon) >= 7) 
+
+                        leh = (c4gli_afternoon.air_balloon.z < c4gli_afternoon.pars.h)
+                        logic_afternoon['afternoon_mlerrlow'] = (\
+                                (len(np.where(leh)[0]) > 0) and \
+                                # in cases where humidity is not defined, the mixed-layer
+                                # values get corr
+                                (not np.isnan(c4gli_afternoon.pars.theta)) and (
+                                # in cases where humidity is not defined, the mixed-layer
+                                (rmse(c4gli_afternoon.air_balloon.theta[leh] , c4gli_afternoon.pars.theta,filternan_actual=True) < 1.5))\
+                                      )
+
+                        # we only store the last afternoon sounding that fits these
+                        # minimum criteria
+
+                        afternoon_ok = np.mean(list(logic_afternoon.values()))
+
+                        #we set the definitive first afternoon logic only the first time. We only set a new one later if we find a good one.
+                        if afternoon_first:
+                            logic_afternoon_def = {**logic_afternoon,**dict()}
+                            afternoon_first = False
+
+                        print('logic_afternoon: ',logic_afternoon)
+                        df_logic_afternoon = df_logic_afternoon.append(logic_afternoon, ignore_index=True)
+                        # if len(df_logic_afternoon)> 30:
+                        #     stop
+                        print(afternoon_ok,c4gli_afternoon.pars.ldatetime)
+                        if afternoon_ok == 1.:
+                            # # doesn't work :(
+                            # c4gli_afternoon_for_dump = cp.deepcopy(c4gli_afternoon)
+                            
+                            # so we just create a new one from the same wyoming profile
+                            c4gli_afternoon_for_dump = class4gl_input()
+                            c4gli_afternoon_for_dump.get_profile_wyoming(wy_strm)
+                            logic_afternoon_def = {**logic_afternoon,**dict()}
+                            afternoon_set = True
+
+                        
+                        wy_strm.find_next()
+                        c4gli_afternoon.clear()
+                        c4gli_afternoon.get_profile_wyoming(wy_strm)
+
+                        if wy_strm.current is not None:
+                            current_date_afternoon = \
+                                   dt.date(c4gli_afternoon.pars.ldatetime.year, \
+                                           c4gli_afternoon.pars.ldatetime.month, \
+                                           c4gli_afternoon.pars.ldatetime.day)
+                        else:
+                            # a dummy date: this will be ignored anyway
+                            current_date_afternoon = dt.date(1900,1,1)
+                    if not afternoon_set:
+                        logic_afternoon_def = {**logic_afternoon,**dict()}
+
+                        # Only in the case we have a good pair of soundings, we
+                        # dump them to disk
+                    logic_afternoon_def['afternoon_ok'] = False
+                    if c4gli_afternoon_for_dump is not None:
+                        logic_afternoon_def['afternoon_ok'] = True
+                        c4gli.update(source='pairs',pars={'runtime' : \
+                            int((c4gli_afternoon_for_dump.pars.datetime_daylight - 
+                                 c4gli.pars.datetime_daylight).total_seconds())})
+    
+    
+                        logic_afternoon_def['runtime_ok'] = False
+                        print('ALMOST...')
+
+                        if c4gli.pars.runtime >= 3600*4.: # more than 4 hours simulation
+                             
+                            logic_afternoon_def['runtime_ok'] = True
+                            logic_afternoon_def['not_polar'] = False
+                            if abs(c4gli.pars.lat) < 70.:
+                                logic_afternoon_def['not_polar'] = True
+                                logic_afternoon_def['global_parameters_ok'] = False
+        
+                                c4gli.get_global_input(globaldata)
+                                #saopaulo:sandyclayloam -> --force_pars "LAI=3.5,b=7.12,CGsat=3.670,p=6,a=0.135,C2ref=0.8,C1sat=0.213"
+                                #curitiba:clay -> --force_pars "LAI=3.5,b=11.40,CGsat=3.6,p=12,a=0.083,C2ref=0.3,C1sat=0.342"
+                                for parkey,parvalue in [par.split('=') for par in args.force_pars.split(',')]:
+                                    c4gli.update(source='user_specified', pars={parkey:float(parvalue)})
+                                print('VERY CLOSE...')
+                                if (args.check_inputdata == 'False') or (c4gli.check_source_globaldata()  and \
+                                    (c4gli.check_source(source='wyoming',\
+                                                       check_only_sections='pars'))):
+
+                                    logic_afternoon_def['global_parameters_ok'] = True
+                                    print('starting dumps')
+                                    c4gli.dump(fileout)
+                                    print('file morning dumped')
+                                    c4gli_afternoon_for_dump.dump(fileout_afternoon)
+                                    print('file afternoon dumped')
+                                    
+                                    
+                                    # for keyEXP,dictEXP in experiments.items():
+                                    #     
+                                    #     c4gli.update(source=keyEXP,pars = dictEXP)
+                                    #     c4gl = class4gl(c4gli)
+                                    #     # c4gl.run()
+                                    #     
+                                    #     c4gl.dump(c4glfiles[key])
+                                    
+                                    print('HIT!!!')
+                                    one_run = True
+                except:
+                    print('get profile failed')
+            dict_diag_day = {}
+
+            dict_diag_day = {**dict_diag_day , **c4gli.pars.__dict__}
+            dict_diag_day = {**dict_diag_day , **logic}
+            dict_diag_day = {**dict_diag_day , **logic_afternoon_def}
+            for key,value in dict_diag_day.items():
+                if key not in dict_diag_station.keys():
+                    dict_diag_station[key] = {}
+                dict_diag_station[key][(c4gli.pars.STNID,c4gli.pars.ldatetime)] = dict_diag_day[key]
+    pd.DataFrame.from_dict(dict_diag_station).to_pickle(fnout_diag)
+                
+    if one_run:
+        #STN.name = STN.name
+        all_records_morning = get_records(pd.DataFrame([STN]),\
+                                      args.path_output,\
+                                      subset='ini',
+                                      refetch_records=True,
+                                      )
+        all_records_afternoon = get_records(pd.DataFrame([STN]),\
+                                      args.path_output,\
+                                      subset='end',
+                                      refetch_records=True,
+                                      )
+    else:
+        print('no valid record found. Removing files', fnout, fnout_afternoon)
+        os.system('rm '+fnout)
+        os.system('rm '+fnout_afternoon)
+
+    # for c4glfile in c4glfiles:
+    #     c4glfile.close()            
+
diff --git a/class4gl/setup/setup_igra_20181217.py b/class4gl/setup/setup_igra_20181217.py
new file mode 100644
index 0000000..03da366
--- /dev/null
+++ b/class4gl/setup/setup_igra_20181217.py
@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thursday, March 29, 11:30 AM
+
+@author: Hendrik Wouters
+
+The dry-2-dry global radio sounding experiment.
+
+usage:
+    python setup_global.py 
+    where  is an integer indicating the row index of the station list
+    under args.path_output+'/'+fn_stations (see below)
+
+this scripts should be called from the pbs script setup_global.pbs
+
+
+
+dependencies:
+    - pandas
+    - class4gl
+    - data_soundings
+
+
+"""
+
+""" import libraries """
+import pandas as pd
+import sys
+#import copy as cp
+import numpy as np
+#from sklearn.metrics import mean_squared_error
+import logging
+import datetime as dt
+import os
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_input')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+# parser.add_argument('--first_YYYYMMDD',default="19810101")
+# parser.add_argument('--last_YYYYMMDD',default="20180101")
+parser.add_argument('--startyear',default="1981")
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--station_id') # run a specific station id
+# parser.add_argument('--error_handling',default='dump_on_success')
+# parser.add_argument('--subset_output',default='morning') # this tells which yaml subset
+
+
+# args.path_output = "/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/GLOBAL/"
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+fn_stations = args.path_input+'/igra-stations.txt'
+
+
+#calculate the root mean square error
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+from class4gl import class4gl_input, data_global,class4gl
+from data_soundings import wyoming
+#from data_global import data_global
+
+# iniitialize global data
+globaldata = data_global()
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+# read the list of stations with valid ground data (list generated with
+# get_valid_stations.py)
+# args.path_input = "/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/"
+
+# df_stations = pd.read_fwf(fn_stations,names=['Country code',\
+#                                                'ID',\
+#                                                'Name',\
+#                                                'latitude',\
+#                                                'longitude',\
+#                                                'height',\
+#                                                'unknown',\
+#                                                'startyear',\
+#                                                'endyear'])
+# 
+
+# ===============================
+print("getting a list of stations")
+# ===============================
+all_stations = stations(args.path_input,refetch_stations=False)
+df_stations = all_stations.table
+df_stations.columns
+
+if args.station_id is not None:
+    df_stations = df_stations.query('STNID == '+args.station_id)
+else:
+    if args.last_station_row is not None:
+        df_stations = df_stations[:(int(args.last_station_row)+1)]
+    if args.first_station_row is not None:
+        df_stations = df_stations[int(args.first_station_row):]
+
+STNlist = list(df_stations.iterrows())
+
+os.system('mkdir -p '+args.path_output)
+for iSTN,STN in STNlist:  
+    one_run = False
+# for iSTN,STN in STNlist[5:]:  
+    
+    fnout = args.path_output+"/"+format(STN.name,'05d')+"_ini.yaml"
+    fnout_afternoon = args.path_output+"/"+format(STN.name,'05d')+"_end.yaml"
+    
+
+    # c4glfiles = dict([(EXP,odirexperiments[EXP]+'/'+format(STN['ID'],'05d')+'.yaml') \
+    #                   for EXP in experiments.keys()])
+        
+    with open(fnout,'w') as fileout, \
+         open(fnout_afternoon,'w') as fileout_afternoon:
+        wy_strm = wyoming(PATH=args.path_input, STNM=STN.name)
+        wy_strm.set_STNM(int(STN.name))
+
+        # we consider all soundings from 1981 onwards
+        wy_strm.find_first(year=int(args.startyear))
+        #wy_strm.find(dt.datetime(2004,10,19,6))
+        
+        c4gli = class4gl_input(debug_level=logging.INFO)
+        c4gli_afternoon = class4gl_input(debug_level=logging.INFO)
+        # so we continue as long as we can find a new sounding
+                
+        while wy_strm.current is not None:
+            
+            c4gli.clear()
+            try: 
+                c4gli.get_profile_wyoming(wy_strm)
+                #print(STN['ID'],c4gli.pars.datetime)
+                #c4gli.get_global_input(globaldata)
+
+                print(c4gli.pars.STNID, c4gli.pars.ldatetime)
+
+                logic = dict()
+                logic['morning'] =  (c4gli.pars.ldatetime.hour <= 12.)
+
+                # Sounding should have taken place after 3 hours before sunrise.
+                # Note that the actual simulation only start at sunrise
+                # (specified by ldatetime_daylight), so the ABL cooling af the time
+                # before sunrise is ignored by the simulation.
+                logic['daylight'] = \
+                    ((c4gli.pars.ldatetime - 
+                      c4gli.pars.lSunrise).total_seconds()/3600. >= -3.)
+                
+                logic['springsummer'] = (c4gli.pars.theta > 278.)
+                
+                # we take 3000 because previous analysis (ie., HUMPPA) has
+                # focussed towards such altitude
+                le3000 = (c4gli.air_balloon.z <= 3000.)
+                logic['10measurements'] = (np.sum(le3000) >= 7) 
+
+                leh = (c4gli.air_balloon.z <= c4gli.pars.h)
+
+                logic['mlerrlow'] = (\
+                        (len(np.where(leh)[0]) > 0) and \
+                        # in cases where humidity is not defined, the mixed-layer
+                        # values get corr
+                        (not np.isnan(c4gli.pars.theta))\
+                                     #and \
+                        #(rmse(c4gli.air_balloon.theta[leh] , \
+                        #      c4gli.pars.theta,filternan_actual=True) < 1.0)\
+                              )
+    
+
+                logic['mlherrlow'] = (c4gli.pars.h_e <= 150.)
+                
+                print('logic:', logic)
+                # the result
+                morning_ok = np.mean(list(logic.values()))
+                print(morning_ok,c4gli.pars.ldatetime)
+
+            except:
+                morning_ok =False
+                print('obtain morning not good')
+
+            # the next sounding will be used either for an afternoon sounding
+            # or for the morning sounding of the next day.
+            wy_strm.find_next()
+            # If the morning is ok, then we try to find a decent afternoon
+            # sounding
+            if morning_ok == 1.:
+                print('MORNING OK!')
+                # we get the current date
+                current_date = dt.date(c4gli.pars.ldatetime.year, \
+                                       c4gli.pars.ldatetime.month, \
+                                       c4gli.pars.ldatetime.day)
+                c4gli_afternoon.clear()
+                print('AFTERNOON PROFILE CLEARED')
+                try:
+                    c4gli_afternoon.get_profile_wyoming(wy_strm)
+                    print('AFTERNOON PROFILE OK')
+
+                    if wy_strm.current is not None:
+                        current_date_afternoon = \
+                                   dt.date(c4gli_afternoon.pars.ldatetime.year, \
+                                           c4gli_afternoon.pars.ldatetime.month, \
+                                           c4gli_afternoon.pars.ldatetime.day)
+                    else:
+                        # a dummy date: this will be ignored anyway
+                        current_date_afternoon = dt.date(1900,1,1)
+
+                    # we will dump the latest afternoon sounding that fits the
+                    # minimum criteria specified by logic_afternoon
+                    print(current_date,current_date_afternoon)
+                    c4gli_afternoon_for_dump = None
+                    while ((current_date_afternoon == current_date) and \
+                           (wy_strm.current is not None)):
+                        logic_afternoon =dict()
+
+                        logic_afternoon['afternoon'] = \
+                            (c4gli_afternoon.pars.ldatetime.hour >= 12.)
+                        # the sounding should have taken place before 1 hours
+                        # before sunset. This is to minimize the chance that a
+                        # stable boundary layer (yielding very low mixed layer
+                        # heights) is formed which can not be represented by
+                        # class.
+                        logic_afternoon['daylight'] = \
+                          ((c4gli_afternoon.pars.ldatetime - \
+                            c4gli_afternoon.pars.lSunset \
+                           ).total_seconds()/3600. <= -1.)
+
+
+                        le3000_afternoon = \
+                            (c4gli_afternoon.air_balloon.z <= 3000.)
+                        logic_afternoon['5measurements'] = \
+                            (np.sum(le3000_afternoon) >= 7) 
+
+                        # we only store the last afternoon sounding that fits these
+                        # minimum criteria
+
+                        afternoon_ok = np.mean(list(logic_afternoon.values()))
+
+                        print('logic_afternoon: ',logic_afternoon)
+                        print(afternoon_ok,c4gli_afternoon.pars.ldatetime)
+                        if afternoon_ok == 1.:
+                            # # doesn't work :(
+                            # c4gli_afternoon_for_dump = cp.deepcopy(c4gli_afternoon)
+                            
+                            # so we just create a new one from the same wyoming profile
+                            c4gli_afternoon_for_dump = class4gl_input()
+                            c4gli_afternoon_for_dump.get_profile_wyoming(wy_strm)
+
+                        wy_strm.find_next()
+                        c4gli_afternoon.clear()
+                        c4gli_afternoon.get_profile_wyoming(wy_strm)
+
+                        if wy_strm.current is not None:
+                            current_date_afternoon = \
+                                   dt.date(c4gli_afternoon.pars.ldatetime.year, \
+                                           c4gli_afternoon.pars.ldatetime.month, \
+                                           c4gli_afternoon.pars.ldatetime.day)
+                        else:
+                            # a dummy date: this will be ignored anyway
+                            current_date_afternoon = dt.date(1900,1,1)
+
+                        # Only in the case we have a good pair of soundings, we
+                        # dump them to disk
+                    if c4gli_afternoon_for_dump is not None:
+                        c4gli.update(source='pairs',pars={'runtime' : \
+                            int((c4gli_afternoon_for_dump.pars.datetime_daylight - 
+                                 c4gli.pars.datetime_daylight).total_seconds())})
+    
+    
+                        print('ALMOST...')
+                        if c4gli.pars.runtime > 3600*4.: # more than 4 hours simulation
+                                
+        
+                            c4gli.get_global_input(globaldata)
+                            print('VERY CLOSE...')
+                            if c4gli.check_source_globaldata() and \
+                                (c4gli.check_source(source='wyoming',\
+                                                   check_only_sections='pars')):
+                                c4gli.dump(fileout)
+                                
+                                c4gli_afternoon_for_dump.dump(fileout_afternoon)
+                                
+                                
+                                # for keyEXP,dictEXP in experiments.items():
+                                #     
+                                #     c4gli.update(source=keyEXP,pars = dictEXP)
+                                #     c4gl = class4gl(c4gli)
+                                #     # c4gl.run()
+                                #     
+                                #     c4gl.dump(c4glfiles[key])
+                                
+                                print('HIT!!!')
+                                one_run = True
+                except:
+                    print('get profile failed')
+                
+    if one_run:
+        #STN.name = STN.name
+        all_records_morning = get_records(pd.DataFrame([STN]),\
+                                      args.path_output,\
+                                      subset='ini',
+                                      refetch_records=True,
+                                      )
+        all_records_afternoon = get_records(pd.DataFrame([STN]),\
+                                      args.path_output,\
+                                      subset='end',
+                                      refetch_records=True,
+                                      )
+    else:
+        os.system('rm '+fnout)
+        os.system('rm '+fnout_afternoon)
+
+    # for c4glfile in c4glfiles:
+    #     c4glfile.close()            
+
diff --git a/class4gl/setup/setup_igra_pkl.py b/class4gl/setup/setup_igra_pkl.py
new file mode 100644
index 0000000..8e795fe
--- /dev/null
+++ b/class4gl/setup/setup_igra_pkl.py
@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thursday, March 29, 11:30 AM
+
+@author: Hendrik Wouters
+
+The dry-2-dry global radio sounding experiment.
+
+usage:
+    python setup_global.py 
+    where  is an integer indicating the row index of the station list
+    under args.path_output+'/'+fn_stations (see below)
+
+this scripts should be called from the pbs script setup_global.pbs
+
+
+
+dependencies:
+    - pandas
+    - class4gl
+    - data_soundings
+
+
+"""
+
+""" import libraries """
+import pandas as pd
+import sys
+#import copy as cp
+import numpy as np
+#from sklearn.metrics import mean_squared_error
+import logging
+import datetime as dt
+import os
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_input')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+# parser.add_argument('--first_YYYYMMDD',default="19810101")
+# parser.add_argument('--last_YYYYMMDD',default="20180101")
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--station_id') # run a specific station id
+# parser.add_argument('--error_handling',default='dump_on_success')
+# parser.add_argument('--subset_output',default='morning') # this tells which yaml subset
+
+
+# args.path_output = "/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/GLOBAL/"
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+
+fn_stations = args.path_input+'/igra-stations.txt'
+
+
+#calculate the root mean square error
+
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    rmse_temp = (y_actual_temp - y_predicted_temp)
+    rmse_temp = np.mean(rmse_temp*rmse_temp)
+    return np.sqrt(rmse_temp)
+
+
+from class4gl import class4gl_input, data_global,class4gl
+from data_soundings import wyoming
+#from data_global import data_global
+
+# iniitialize global data
+globaldata = data_global()
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+# read the list of stations with valid ground data (list generated with
+# get_valid_stations.py)
+# args.path_input = "/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/"
+
+# df_stations = pd.read_fwf(fn_stations,names=['Country code',\
+#                                                'ID',\
+#                                                'Name',\
+#                                                'latitude',\
+#                                                'longitude',\
+#                                                'height',\
+#                                                'unknown',\
+#                                                'startyear',\
+#                                                'endyear'])
+# 
+
+# ===============================
+print("getting a list of stations")
+# ===============================
+all_stations = stations(args.path_input,refetch_stations=False)
+df_stations = all_stations.table
+df_stations.columns
+
+if args.station_id is not None:
+    df_stations = df_stations.query('STNID == '+args.station_id)
+else:
+    if args.last_station_row is not None:
+        df_stations = df_stations[:(int(args.last_station_row)+1)]
+    if args.first_station_row is not None:
+        df_stations = df_stations[int(args.first_station_row):]
+
+STNlist = list(df_stations.iterrows())
+
+os.system('mkdir -p '+args.path_output)
+for iSTN,STN in STNlist:  
+    one_run = True
+# for iSTN,STN in STNlist[5:]:  
+    
+    fnout = args.path_output+"/"+format(STN.name,'05d')+"_morning.yaml"
+    fnout_afternoon = args.path_output+"/"+format(STN.name,'05d')+"_afternoon.yaml"
+    
+
+    # # c4glfiles = dict([(EXP,odirexperiments[EXP]+'/'+format(STN['ID'],'05d')+'.yaml') \
+    # #                   for EXP in experiments.keys()])
+    #     
+    # with open(fnout,'w') as fileout, \
+    #      open(fnout_afternoon,'w') as fileout_afternoon:
+    #     wy_strm = wyoming(PATH=args.path_input, STNM=STN.name)
+    #     wy_strm.set_STNM(int(STN.name))
+
+    #     # we consider all soundings from 1981 onwards
+    #     wy_strm.find_first(year=1981)
+    #     #wy_strm.find(dt.datetime(2004,10,19,6))
+    #     
+    #     c4gli = class4gl_input(debug_level=logging.INFO)
+    #     c4gli_afternoon = class4gl_input(debug_level=logging.INFO)
+    #     # so we continue as long as we can find a new sounding
+    #             
+    #     while wy_strm.current is not None:
+    #         
+    #         c4gli.clear()
+    #         try: 
+    #             c4gli.get_profile_wyoming(wy_strm)
+    #             #print(STN['ID'],c4gli.pars.datetime)
+    #             #c4gli.get_global_input(globaldata)
+
+    #             print(c4gli.pars.STNID, c4gli.pars.ldatetime)
+
+    #             logic = dict()
+    #             logic['morning'] =  (c4gli.pars.ldatetime.hour <= 12.)
+
+    #             # Sounding should have taken place after 3 hours before sunrise.
+    #             # Note that the actual simulation only start at sunrise
+    #             # (specified by ldatetime_daylight), so the ABL cooling af the time
+    #             # before sunrise is ignored by the simulation.
+    #             logic['daylight'] = \
+    #                 ((c4gli.pars.ldatetime - 
+    #                   c4gli.pars.lSunrise).total_seconds()/3600. >= -3.)
+    #             
+    #             logic['springsummer'] = (c4gli.pars.theta > 278.)
+    #             
+    #             # we take 3000 because previous analysis (ie., HUMPPA) has
+    #             # focussed towards such altitude
+    #             le3000 = (c4gli.air_balloon.z <= 3000.)
+    #             logic['10measurements'] = (np.sum(le3000) >= 10) 
+
+    #             leh = (c4gli.air_balloon.z <= c4gli.pars.h)
+
+    #             logic['mlerrlow'] = (\
+    #                     (len(np.where(leh)[0]) > 0) and \
+    #                     # in cases where humidity is not defined, the mixed-layer
+    #                     # values get corr
+    #                     (not np.isnan(c4gli.pars.theta)) and \
+    #                     (rmse(c4gli.air_balloon.theta[leh] , \
+    #                           c4gli.pars.theta,filternan_actual=True) < 1.)\
+    #                           )
+    # 
+
+    #             logic['mlherrlow'] = (c4gli.pars.h_e <= 150.)
+    #             
+    #             print('logic:', logic)
+    #             # the result
+    #             morning_ok = np.mean(list(logic.values()))
+    #             print(morning_ok,c4gli.pars.ldatetime)
+
+    #         except:
+    #             morning_ok =False
+    #             print('obtain morning not good')
+
+    #         # the next sounding will be used either for an afternoon sounding
+    #         # or for the morning sounding of the next day.
+    #         wy_strm.find_next()
+    #         # If the morning is ok, then we try to find a decent afternoon
+    #         # sounding
+    #         if morning_ok == 1.:
+    #             print('MORNING OK!')
+    #             # we get the current date
+    #             current_date = dt.date(c4gli.pars.ldatetime.year, \
+    #                                    c4gli.pars.ldatetime.month, \
+    #                                    c4gli.pars.ldatetime.day)
+    #             c4gli_afternoon.clear()
+    #             print('AFTERNOON PROFILE CLEARED')
+    #             try:
+    #                 c4gli_afternoon.get_profile_wyoming(wy_strm)
+    #                 print('AFTERNOON PROFILE OK')
+
+    #                 if wy_strm.current is not None:
+    #                     current_date_afternoon = \
+    #                                dt.date(c4gli_afternoon.pars.ldatetime.year, \
+    #                                        c4gli_afternoon.pars.ldatetime.month, \
+    #                                        c4gli_afternoon.pars.ldatetime.day)
+    #                 else:
+    #                     # a dummy date: this will be ignored anyway
+    #                     current_date_afternoon = dt.date(1900,1,1)
+
+    #                 # we will dump the latest afternoon sounding that fits the
+    #                 # minimum criteria specified by logic_afternoon
+    #                 print(current_date,current_date_afternoon)
+    #                 c4gli_afternoon_for_dump = None
+    #                 while ((current_date_afternoon == current_date) and \
+    #                        (wy_strm.current is not None)):
+    #                     logic_afternoon =dict()
+
+    #                     logic_afternoon['afternoon'] = \
+    #                         (c4gli_afternoon.pars.ldatetime.hour >= 12.)
+    #                     # the sounding should have taken place before 2 hours
+    #                     # before sunset. This is to minimize the change that a
+    #                     # stable boundary layer (yielding very low mixed layer
+    #                     # heights) is formed which can not be represented by
+    #                     # class.
+    #                     logic_afternoon['daylight'] = \
+    #                       ((c4gli_afternoon.pars.ldatetime - \
+    #                         c4gli_afternoon.pars.lSunset \
+    #                        ).total_seconds()/3600. <= -2.)
+
+
+    #                     le3000_afternoon = \
+    #                         (c4gli_afternoon.air_balloon.z <= 3000.)
+    #                     logic_afternoon['5measurements'] = \
+    #                         (np.sum(le3000_afternoon) >= 5) 
+
+    #                     # we only store the last afternoon sounding that fits these
+    #                     # minimum criteria
+
+    #                     afternoon_ok = np.mean(list(logic_afternoon.values()))
+
+    #                     print('logic_afternoon: ',logic_afternoon)
+    #                     print(afternoon_ok,c4gli_afternoon.pars.ldatetime)
+    #                     if afternoon_ok == 1.:
+    #                         # # doesn't work :(
+    #                         # c4gli_afternoon_for_dump = cp.deepcopy(c4gli_afternoon)
+    #                         
+    #                         # so we just create a new one from the same wyoming profile
+    #                         c4gli_afternoon_for_dump = class4gl_input()
+    #                         c4gli_afternoon_for_dump.get_profile_wyoming(wy_strm)
+
+    #                     wy_strm.find_next()
+    #                     c4gli_afternoon.clear()
+    #                     c4gli_afternoon.get_profile_wyoming(wy_strm)
+
+    #                     if wy_strm.current is not None:
+    #                         current_date_afternoon = \
+    #                                dt.date(c4gli_afternoon.pars.ldatetime.year, \
+    #                                        c4gli_afternoon.pars.ldatetime.month, \
+    #                                        c4gli_afternoon.pars.ldatetime.day)
+    #                     else:
+    #                         # a dummy date: this will be ignored anyway
+    #                         current_date_afternoon = dt.date(1900,1,1)
+
+    #                     # Only in the case we have a good pair of soundings, we
+    #                     # dump them to disk
+    #                 if c4gli_afternoon_for_dump is not None:
+    #                     c4gli.update(source='pairs',pars={'runtime' : \
+    #                         int((c4gli_afternoon_for_dump.pars.datetime_daylight - 
+    #                              c4gli.pars.datetime_daylight).total_seconds())})
+    # 
+    # 
+    #                     print('ALMOST...')
+    #                     if c4gli.pars.runtime > 18000.: # more than 5 hours simulation
+    #                             
+    #     
+    #                         c4gli.get_global_input(globaldata)
+    #                         print('VERY CLOSE...')
+    #                         if c4gli.check_source_globaldata() and \
+    #                             (c4gli.check_source(source='wyoming',\
+    #                                                check_only_sections='pars')):
+    #                             c4gli.dump(fileout)
+    #                             
+    #                             c4gli_afternoon_for_dump.dump(fileout_afternoon)
+    #                             
+    #                             
+    #                             # for keyEXP,dictEXP in experiments.items():
+    #                             #     
+    #                             #     c4gli.update(source=keyEXP,pars = dictEXP)
+    #                             #     c4gl = class4gl(c4gli)
+    #                             #     # c4gl.run()
+    #                             #     
+    #                             #     c4gl.dump(c4glfiles[key])
+    #                             
+    #                             print('HIT!!!')
+    #                             one_run = True
+    #             except:
+    #                 print('get profile failed')
+    #             
+    if one_run:
+        #STN.name = STN.name
+        all_records_morning = get_records(pd.DataFrame([STN]),\
+                                      args.path_output,\
+                                      subset='morning',
+                                      refetch_records=True,
+                                      )
+        all_records_afternoon = get_records(pd.DataFrame([STN]),\
+                                      args.path_output,\
+                                      subset='afternoon',
+                                      refetch_records=True,
+                                      )
+    # else:
+    #     os.system('rm '+fnout)
+    #     os.system('rm '+fnout_afternoon)
+
+    # for c4glfile in c4glfiles:
+    #     c4glfile.close()            
+
diff --git a/class4gl/setup/trash/setup_global_old.py b/class4gl/setup/trash/setup_global_old.py
new file mode 100644
index 0000000..d812684
--- /dev/null
+++ b/class4gl/setup/trash/setup_global_old.py
@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thursday, March 29, 11:30 AM
+
+@author: Hendrik Wouters
+
+The dry-2-dry global radio sounding experiment.
+
+usage:
+    python setup_global.py 
+    where  is an integer indicating the row index of the station list
+    under odir+'/'+fn_stations (see below)
+
+this scripts should be called from the pbs script setup_global.pbs
+
+
+
+dependencies:
+    - pandas
+    - class4gl
+    - data_soundings
+
+
+"""
+
+""" import libraries """
+import pandas as pd
+import sys
+#import copy as cp
+import numpy as np
+from sklearn.metrics import mean_squared_error
+import logging
+import datetime as dt
+import os
+import math
+
+odir = "/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/GLOBAL/"
+fn_stations = odir+'/igra-stations_sel.txt'
+
+
+#calculate the root mean square error
+def rmse(y_actual,y_predicted,z_actual = None, z_predicted = None,filternan_actual = False):
+    """ calculated root mean squared error 
+        
+    
+        INPUT:
+            y_actual: reference dataset
+            y_predicted: predicting dataset
+            z_actual: coordinate values of reference dataset
+            z_predicted: coordinate values of the predicting dataset
+            
+            filternan_actual: throw away reference values that have nans
+    """
+    
+    y_actual_temp = np.array(y_actual)
+    y_predicted_temp = np.array(y_predicted)
+    
+    if z_actual is not None:
+        z_actual_temp = np.array(z_actual)
+    else: 
+        z_actual_temp = None
+        
+    
+    if filternan_actual:
+        y_actual_temp = y_actual_temp[~np.isnan(y_actual_temp)]
+        if z_actual_temp is not None:
+            z_actual_temp = z_actual_temp[~np.isnan(y_actual_temp)]
+    
+    if ((z_actual_temp is not None) or (z_predicted is not None)):    
+        if (z_actual_temp is None) or (z_predicted is None):
+            raise ValueError('Input z_actual and z_predicted need \
+                              to be specified simultaneously.')
+        y_predicted_temp = np.interp(z_actual_temp,z_predicted, y_predicted)
+    
+    else:
+        # this catches the situation that y_predicted is a single value (eg., 
+        # which is the case for evaluating eg., mixed-layer estimates)
+        y_predicted_temp = y_actual_temp*0. + y_predicted_temp
+        
+    
+    return np.sqrt(mean_squared_error(y_actual_temp,y_predicted_temp))
+
+
+sys.path.insert(0, '/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/')
+from class4gl import class4gl_input, data_global,class4gl
+from data_soundings import wyoming
+#from data_global import data_global
+
+# iniitialize global data
+globaldata = data_global()
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+# read the list of stations with valid ground data (list generated with
+# get_valid_stations.py)
+idir = "/user/data/gent/gvo000/gvo00090/EXT/data/SOUNDINGS/"
+
+df_stations = pd.read_csv(fn_stations)
+
+
+STNlist = list(df_stations.iterrows())
+NUMSTNS = len(STNlist)
+PROCS = 100
+BATCHSIZE = 1 #math.ceil(np.float(NUMSTNS)/np.float(PROCS))
+
+
+iPROC = int(sys.argv[1])
+
+
+for iSTN,STN in STNlist[iPROC*BATCHSIZE:(iPROC+1)*BATCHSIZE]:  
+# for iSTN,STN in STNlist[5:]:  
+    
+    fnout = odir+"/"+format(STN['ID'],'05d')+"_morning.yaml"
+    fnout_afternoon = odir+"/"+format(STN['ID'],'05d')+"_afternoon.yaml"
+    
+
+    # c4glfiles = dict([(EXP,odirexperiments[EXP]+'/'+format(STN['ID'],'05d')+'.yaml') \
+    #                   for EXP in experiments.keys()])
+        
+    with open(fnout,'w') as fileout, \
+         open(fnout_afternoon,'w') as fileout_afternoon:
+        wy_strm = wyoming(PATH=idir, STNM=STN['ID'])
+        wy_strm.set_STNM(int(STN['ID']))
+
+        # we consider all soundings after 1981
+        wy_strm.find_first(year=1981)
+        #wy_strm.find(dt.datetime(2004,10,19,6))
+        
+        c4gli = class4gl_input(debug_level=logging.INFO)
+        c4gli_afternoon = class4gl_input(debug_level=logging.INFO)
+        # so we continue as long as we can find a new sounding
+        while wy_strm.current is not None:
+            
+            c4gli.clear()
+            c4gli.get_profile_wyoming(wy_strm)
+            #print(STN['ID'],c4gli.pars.datetime)
+            #c4gli.get_global_input(globaldata)
+
+            print(c4gli.pars.STNID, c4gli.pars.ldatetime)
+
+            logic = dict()
+            logic['morning'] =  (c4gli.pars.ldatetime.hour < 12.)
+            logic['daylight'] = \
+                ((c4gli.pars.ldatetime_daylight - 
+                  c4gli.pars.ldatetime).total_seconds()/3600. <= 5.)
+            
+            logic['springsummer'] = (c4gli.pars.theta > 278.)
+            
+            # we take 3000 because previous analysis (ie., HUMPPA) has
+            # focussed towards such altitude
+            le3000 = (c4gli.air_balloon.z <= 3000.)
+            logic['10measurements'] = (np.sum(le3000) >= 10) 
+
+            leh = (c4gli.air_balloon.z <= c4gli.pars.h)
+
+            try:
+                logic['mlerrlow'] = (\
+                        (len(np.where(leh)[0]) > 0) and \
+                        # in cases where humidity is not defined, the mixed-layer
+                        # values get corr
+                        (not np.isnan(c4gli.pars.theta)) and \
+                        (rmse(c4gli.air_balloon.theta[leh] , \
+                              c4gli.pars.theta,filternan_actual=True) < 1.)\
+                              )
+    
+            except:
+                logic['mlerrlow'] = False
+                print('rmse probably failed')
+
+            logic['mlherrlow'] = (c4gli.pars.h_e <= 150.)
+            
+            print('logic:', logic)
+            # the result
+            morning_ok = np.mean(list(logic.values()))
+            print(morning_ok,c4gli.pars.ldatetime)
+            
+            # the next sounding will be used either for an afternoon sounding
+            # or for the morning sounding of the next day.
+            wy_strm.find_next()
+
+            # If the morning is ok, then we try to find a decent afternoon
+            # sounding
+            if morning_ok == 1.:
+                # we get the current date
+                current_date = dt.date(c4gli.pars.ldatetime.year, \
+                                       c4gli.pars.ldatetime.month, \
+                                       c4gli.pars.ldatetime.day)
+                c4gli_afternoon.clear()
+                c4gli_afternoon.get_profile_wyoming(wy_strm)
+
+                if wy_strm.current is not None:
+                    current_date_afternoon = \
+                               dt.date(c4gli_afternoon.pars.ldatetime.year, \
+                                       c4gli_afternoon.pars.ldatetime.month, \
+                                       c4gli_afternoon.pars.ldatetime.day)
+                else:
+                    # a dummy date: this will be ignored anyway
+                    current_date_afternoon = dt.date(1900,1,1)
+
+                # we will dump the latest afternoon sounding that fits the
+                # minimum criteria specified by logic_afternoon
+                c4gli_afternoon_for_dump = None
+                while ((current_date_afternoon == current_date) and \
+                       (wy_strm.current is not None)):
+                    logic_afternoon =dict()
+
+                    logic_afternoon['afternoon'] = \
+                        (c4gli_afternoon.pars.ldatetime.hour >= 12.)
+                    logic_afternoon['daylight'] = \
+                      ((c4gli_afternoon.pars.ldatetime - \
+                        c4gli_afternoon.pars.ldatetime_daylight \
+                       ).total_seconds()/3600. <= 2.)
+
+
+                    le3000_afternoon = \
+                        (c4gli_afternoon.air_balloon.z <= 3000.)
+                    logic_afternoon['5measurements'] = \
+                        (np.sum(le3000_afternoon) >= 5) 
+
+                    # we only store the last afternoon sounding that fits these
+                    # minimum criteria
+
+                    afternoon_ok = np.mean(list(logic_afternoon.values()))
+
+                    print('logic_afternoon: ',logic_afternoon)
+                    print(afternoon_ok,c4gli_afternoon.pars.ldatetime)
+                    if afternoon_ok == 1.:
+                        # # doesn't work :(
+                        # c4gli_afternoon_for_dump = cp.deepcopy(c4gli_afternoon)
+                        
+                        # so we just create a new one from the same wyoming profile
+                        c4gli_afternoon_for_dump = class4gl_input()
+                        c4gli_afternoon_for_dump.get_profile_wyoming(wy_strm)
+
+                    wy_strm.find_next()
+                    c4gli_afternoon.clear()
+                    c4gli_afternoon.get_profile_wyoming(wy_strm)
+
+                    if wy_strm.current is not None:
+                        current_date_afternoon = \
+                               dt.date(c4gli_afternoon.pars.ldatetime.year, \
+                                       c4gli_afternoon.pars.ldatetime.month, \
+                                       c4gli_afternoon.pars.ldatetime.day)
+                    else:
+                        # a dummy date: this will be ignored anyway
+                        current_date_afternoon = dt.date(1900,1,1)
+
+                    # Only in the case we have a good pair of soundings, we
+                    # dump them to disk
+                if c4gli_afternoon_for_dump is not None:
+                    c4gli.update(source='pairs',pars={'runtime' : \
+                        int((c4gli_afternoon_for_dump.pars.datetime_daylight - 
+                             c4gli.pars.datetime_daylight).total_seconds())})
+    
+    
+                    print('ALMOST...')
+                    if c4gli.pars.runtime > 18000.: # more than 5 hours simulation
+                            
+        
+                        c4gli.get_global_input(globaldata)
+                        print('VERY CLOSE...')
+                        if c4gli.check_source_globaldata() and \
+                            (c4gli.check_source(source='wyoming',\
+                                               check_only_sections='pars')):
+                            c4gli.dump(fileout)
+                            
+                            c4gli_afternoon_for_dump.dump(fileout_afternoon)
+                            
+                            
+                            # for keyEXP,dictEXP in experiments.items():
+                            #     
+                            #     c4gli.update(source=keyEXP,pars = dictEXP)
+                            #     c4gl = class4gl(c4gli)
+                            #     # c4gl.run()
+                            #     
+                            #     c4gl.dump(c4glfiles[key])
+                            
+                            print('HIT!!!')
+                
+                
+    # for c4glfile in c4glfiles:
+    #     c4glfile.close()            
+
diff --git a/class4gl/setup/update_input.py b/class4gl/setup/update_input.py
new file mode 100644
index 0000000..9f1a894
--- /dev/null
+++ b/class4gl/setup/update_input.py
@@ -0,0 +1,339 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_input')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_output')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--updates')
+parser.add_argument('--global_vars')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--error_handling',default='dump_on_success')
+parser.add_argument('--diag_tropo',default=None)#['advt','advq','advu','advv'])
+parser.add_argument('--subset_input',default='ini') # this tells which yaml subset
+                                                      # to initialize with.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+parser.add_argument('--subset_output',default='ini')
+
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+
+parser.add_argument('--experiments')
+parser.add_argument('--split_by',default=-1)# station soundings are split
+                                            # up in chunks
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+
+
+# iniitialize global data
+globaldata = data_global()
+if (args.updates is not None) and ('era_profiles' in args.updates.strip().split(",")):
+    globaldata.sources = {**globaldata.sources,**{
+            "ERAINT:t"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/t_6hourly/t_*_6hourly.nc",
+            "ERAINT:q"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/q_6hourly/q_*_6hourly.nc",
+            "ERAINT:u"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/u_6hourly/u_*_6hourly.nc",
+            "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v_*_6hourly.nc",
+            "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v_*_6hourly.nc",
+            "ERAINT:v"     : "/user/data/gent/gvo000/gvo00090/EXT/data/ERA-INTERIM/by_var_nc/v_6hourly/v_*_6hourly.nc",
+            "ERA5:sshf" : '/user/data/gent/gvo000/gvo00090/EXT/data/ERA5/by_var_nc/slhf_1hourly/slhf_*_1hourly.nc',
+            "ERA5:slhf" : '/user/data/gent/gvo000/gvo00090/EXT/data/ERA5/by_var_nc/sshf_1hourly/sshf_*_1hourly.nc',
+            }}
+
+# ...  and load initial data pages
+globaldata.load_datasets(recalc=0)
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+EXP_DEFS  =\
+{
+  'ERA_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_ERA_NEW':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+    'GLOBAL_ADV_SHR':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False,'sw_shr':True},
+  'GLOBAL_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'IOPS_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'IOPS_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+# ========================
+print("getting a list of stations")
+# ========================
+
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_input,suffix=args.subset_input,refetch_stations=False)
+
+# ====================================
+print('defining all_stations_select')
+# ====================================
+
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+
+
+
+if args.station_id is not None:
+    print("Selecting station by ID")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print(all_stations_select)
+print("getting all records of the whole batch")
+all_records_input_select = get_records(all_stations_select,\
+                                         args.path_input,\
+                                         subset=args.subset_input,
+                                         refetch_records=False,
+                                         )
+
+# only run a specific chunck from the selection
+if args.global_chunk_number is not None:
+    if args.station_chunk_number is not None:
+        raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+
+
+    if not (int(args.split_by) > 0) :
+            raise ValueError("global_chunk_number is specified, but --split_by is not a strict positive number, so I don't know how to split the batch into chunks.")
+
+    run_station_chunk = None
+    print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+    totalchunks = 0
+    stations_iter = all_stations_select.iterrows()
+    in_current_chunk = False
+    try:
+        while not in_current_chunk:
+            istation,current_station = stations_iter.__next__()
+            all_records_input_station_select = all_records_input_select.query('STNID == '+str(current_station.name))
+            chunks_current_station = math.ceil(float(len(all_records_input_station_select))/float(args.split_by))
+            print('chunks_current_station',chunks_current_station)
+            in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+        
+            if in_current_chunk:
+                run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                run_station_chunk = int(args.global_chunk_number) - totalchunks 
+        
+            totalchunks +=chunks_current_station
+        
+
+    except StopIteration:
+       raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+    print("station = ",list(run_stations.index))
+    print("station chunk number:",run_station_chunk)
+
+# if no global chunk is specified, then run the whole station selection in one run, or
+# a specific chunk for each selected station according to # args.station_chunk_number
+else:
+    run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+    if args.station_chunk_number is not None:
+        run_station_chunk = int(args.station_chunk_number)
+        print("station(s) that is processed.",list(run_stations.index))
+        print("chunk number: ",run_station_chunk)
+    else:
+        if args.split_by != -1:
+            raise ValueError("Chunks are defined by --split-by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split-by.")
+        run_station_chunk = 0
+        print("stations that are processed.",list(run_stations.index))
+        
+
+#print(all_stations)
+print('Fetching initial/forcing records')
+records_input = get_records(run_stations,\
+                              args.path_input,\
+                              subset=args.subset_input,
+                              refetch_records=False,
+                              )
+
+
+
+os.system('mkdir -p '+args.path_output)
+for istation,current_station in run_stations.iterrows():
+    print(istation,current_station)
+    records_input_station = records_input.query('STNID == '+str(current_station.name))
+    if (int(args.split_by) * int(run_station_chunk)) >= (len(records_input_station)):
+        print("warning: outside of profile number range for station "+\
+              str(current_station)+". Skipping chunk number for this station.")
+    else:
+        fn_input = args.path_input+'/'+format(current_station.name,'05d')+'_'+args.subset_input+'.yaml'
+        if os.path.isfile(fn_input):
+            file_input = open(fn_input)
+        else:
+            fn_input = \
+                 args.path_input+'/'+format(current_station.name,'05d')+\
+                 '_'+str(run_station_chunk)+'_'+args.subset_input+'.yaml'
+            file_input = open(fn_input)
+
+        fn_output = args.path_output+'/'+format(current_station.name,'05d')+'_'+\
+                 str(int(run_station_chunk))+'_'+args.subset_output+'.yaml'
+        file_output = open(fn_output,'w')
+
+        #iexp = 0
+        onerun = False
+        print('starting station chunk number: '\
+              +str(run_station_chunk)+'(size: '+str(args.split_by)+' soundings)')
+
+        records_input_station_chunk = records_input_station.iloc[((run_station_chunk)*int(args.split_by)):((run_station_chunk+1)*int(args.split_by))] #  [(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+
+        isim = 0
+        for (STNID,chunk,index),record_input in records_input_station_chunk.iterrows():
+                print('starting '+str(isim+1)+' out of '+\
+                  str(len(records_input_station_chunk) )+\
+                  ' (station total: ',str(len(records_input_station)),')')  
+            
+        
+                c4gli_output = get_record_yaml(file_input, 
+                                                record_input.index_start, 
+                                                record_input.index_end,
+                                                mode='ini')
+                if args.global_vars is not None:
+                    c4gli_output.get_global_input(globaldata,only_keys=args.global_vars.strip().split(','))
+
+                if args.diag_tropo is not None:
+                    print('add tropospheric parameters on advection and subsidence (for diagnosis)')
+                    seltropo = (c4gli_output.air_ac.p > c4gli_output.air_ac.p.iloc[-1]+ 3000.*(- 1.2 * 9.81 ))
+                    profile_tropo = c4gli_output.air_ac[seltropo]
+                    for var in args.diag_tropo:#['t','q','u','v',]:
+                        if var[:3] == 'adv':
+                            mean_adv_tropo = np.mean(profile_tropo[var+'_x']+profile_tropo[var+'_y'] )
+                            c4gli_output.update(source='era-interim',pars={var+'_tropo':mean_adv_tropo})
+                        else:
+                            print("warning: tropospheric variable "+var+" not recognized")
+
+
+                if (args.updates is not None) and ('era_profiles' in args.updates.strip().split(",")):
+                    c4gli_output.get_global_input(globaldata,only_keys=['t','u','v','q','sp'])
+
+                    c4gli_output.update(source='era-interim',pars={'Ps' : c4gli_output.pars.sp})
+
+                    cp         = 1005.                 # specific heat of dry air [J kg-1 K-1]
+                    Rd         = 287.                  # gas constant for dry air [J kg-1 K-1]
+                    Rv         = 461.5                 # gas constant for moist air [J kg-1 K-1]
+                    R = (Rd*(1.-c4gli_output.air_ac.q) + Rv*c4gli_output.air_ac.q)
+                    rho = c4gli_output.air_ac.p/R/c4gli_output.air_ac.t
+                    dz = c4gli_output.air_ac.delpdgrav/rho
+                    z = [dz.iloc[-1]/2.]
+                    for idz in list(reversed(range(0,len(dz)-1,1))):
+                        z.append(z[-1]+ (dz[idz+1]+dz[idz])/2.)
+                    z = list(reversed(z))
+
+                    theta = c4gli_output.air_ac.t * \
+                               (c4gli_output.pars.sp/(c4gli_output.air_ac.p))**(R/cp)
+                    thetav   = theta*(1. + 0.61 * c4gli_output.air_ac.q)
+
+                    
+                    c4gli_output.update(source='era-interim',air_ac=pd.DataFrame({'z':list(z),
+                                                                           'theta':list(theta),
+                                                                           'thetav':list(thetav),
+                                                                          }))
+                    air_ap_input = c4gli_output.air_ac[::-1].reset_index().drop('index',axis=1)
+                    air_ap_mode = 'b'
+                    air_ap_input_source = c4gli_output.query_source('air_ac:theta')
+
+
+                    c4gli_output.mixed_layer_fit(air_ap=air_ap_input,
+                                         source=air_ap_input_source,
+                                         mode=air_ap_mode)
+
+
+                onerun = True
+                
+                c4gli_output.dump(file_output)
+                    
+                    
+        file_output.close()
+        file_input.close()
+
+        if onerun:
+            records_output = get_records(pd.DataFrame([current_station]),\
+                                                       args.path_output,\
+                                                       getchunk = int(run_station_chunk),\
+                                                       subset=args.subset_output,
+                                                       refetch_records=True,
+                                                       )
+        else:
+            # remove empty files
+            os.system('rm '+fn_output)
+
+# # align afternoon records with initial records, and set same index
+# records_afternoon.index = records_afternoon.ldatetime.dt.date
+# records_afternoon = records_afternoon.loc[records_output.ldatetime.dt.date]
+# records_afternoon.index = records_output.index
+
+# stations_for_iter = stations(path_exp)
+# for STNID,station in stations_iterator(stations_for_iter):
+#     records_current_station_index = \
+#             (records_output.index.get_level_values('STNID') == STNID)
+#     file_current_station_mod = STNID
+# 
+#     with \
+#     open(path_exp+'/'+format(STNID,"05d")+'_output.yaml','r') as file_station_output, \
+#     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+#     open(path_input+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+#         for (STNID,index),record_output in records_iterator(records_output):
+#             c4gli_output = get_record_yaml(file_station_output, 
+#                                         record_output.index_start, 
+#                                         record_output.index_end,
+#                                         mode='ini')
+#             #print('c4gli_in_ldatetime 3',c4gli_output.pars.ldatetime)
+# 
+#             record_mod = records_mod.loc[(STNID,index)]
+#             c4gl_mod = get_record_yaml(file_station_mod, 
+#                                         record_mod.index_start, 
+#                                         record_mod.index_end,
+#                                         mode='mod')
+#             record_afternoon = records_afternoon.loc[(STNID,index)]
+#             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+#                                         record_afternoon.index_start, 
+#                                         record_afternoon.index_end,
+#                                         mode='ini')
+
diff --git a/class4gl/simulations/batch_simulations.pbs b/class4gl/simulations/batch_simulations.pbs
new file mode 100644
index 0000000..2426063
--- /dev/null
+++ b/class4gl/simulations/batch_simulations.pbs
@@ -0,0 +1,34 @@
+#!/bin/bash 
+#
+#PBS -j oe
+#PBS -M hendrik.wouters@ugent.be
+#PBS -m b
+#PBS -m e
+#PBS -m a
+#PBS -N c4gl_sim
+
+module purge
+
+# echo loading modules: $LOADDEPSCLASS4GL 
+# $LOADDEPSCLASS4GL 
+source ~/load_anaconda.sh
+
+EXEC_ALL="python $C4GLJOB_exec --global_chunk_number $PBS_ARRAYID"
+
+for var in $(compgen -v | grep C4GLJOB_ ); do
+    echo $var
+    if [ "$var" != "C4GLJOB_exec" ]
+    then
+    EXEC_ALL=$EXEC_ALL" --"`echo $var | cut -c9-`"="${!var}
+    fi
+done
+
+
+# EXEC_ALL="python $exec --global-chunk-number $PBS_ARRAYID \
+#                        --split-by $split_by \
+#                        --dataset $dataset \
+#                        --experiments $experiments"
+#                  #      --path-soundings $path_soundings \
+echo Executing: $EXEC_ALL
+$EXEC_ALL
+
diff --git a/class4gl/simulations/batch_simulations.py b/class4gl/simulations/batch_simulations.py
new file mode 100644
index 0000000..fec9032
--- /dev/null
+++ b/class4gl/simulations/batch_simulations.py
@@ -0,0 +1,232 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+from time import sleep
+
+parser = argparse.ArgumentParser()
+#if __name__ == '__main__':
+parser.add_argument('--exec') # chunk simulation script
+parser.add_argument('--first_station_row',
+                    help='starting row number of stations table')
+parser.add_argument('--last_station_row')
+parser.add_argument('--pbs_string',default=' -l walltime=2:0:0')
+parser.add_argument('--station_id',
+                    help="process a specific station id")
+parser.add_argument('--error_handling')
+parser.add_argument('--multi_processing_mode',default='pythonpool')
+parser.add_argument('--cpu_count',type=int,default=2)
+parser.add_argument('--subset_forcing',default='ini') 
+# parser.add_argument('--path_timeseries_forcing',default=False) 
+# parser.add_argument('--timeseries_forcing',default=False) 
+#                                         # this tells which yaml subset
+#                                         # to initialize with.
+#                                         # Most common options are
+#                                         # 'morning' and 'ini'.
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime',
+                    help="set the runtime of the simulation in seconds, or get it from the daytime difference in the profile pairs 'from_profile_pair' (default)")
+# delete folders of experiments before running them
+parser.add_argument('--cleanup_output_directories',
+                    default="False",
+                    help="clean up output directories before executing the experiments")
+parser.add_argument('--experiments', 
+                    help="IDs of experiments, as a space-seperated list (default: 'BASE')")
+parser.add_argument('--experiments_names', 
+                    help="Alternative output names that are given to the experiments. By default, these are the same as --experiments") 
+parser.add_argument('--split_by',
+                    default=50,
+                    type=int,
+                    help="the maxmimum number of soundings that are contained in each output file of a station. -1 means unlimited. The default for array experiments is 50.")
+
+parser.add_argument('--c4gl_path_lib',help="the path of the CLASS4GL program.")#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--path_forcing',
+                    help='directory of forcing data to initialize and constrain the ABL model simulations'
+                   )
+parser.add_argument('--path_experiments',
+                    help='output directory in which the experiments as subdirectories are stored')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+
+
+#arguments only used for update_yaml.py
+#parser.add_argument('--path_dataset') 
+#parser.add_argument('--global_keys') 
+batch_args = parser.parse_args()
+
+if batch_args.c4gl_path_lib is not None:
+    sys.path.insert(0, batch_args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+
+# #SET = 'GLOBAL'
+# SET = batch_args.dataset
+
+# path_forcingSET = batch_args.path_forcing+'/'+SET+'/'
+
+print("getting all stations from "+batch_args.path_forcing)
+# these are all the stations that are found in the input dataset
+all_stations = stations(batch_args.path_forcing,suffix=batch_args.subset_forcing,refetch_stations=False)
+
+print('defining all_stations_select')
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+if batch_args.station_id is not None:
+    print("Selecting stations by --station_id")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(batch_args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table [--first_station_row,--last_station_row]")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if batch_args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(batch_args.last_station)+1)]
+    if batch_args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(batch_args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         batch_args.path_forcing,\
+                                         subset=batch_args.subset_forcing,\
+                                         refetch_records=False,\
+                                        )
+
+print('splitting batch in --split_by='+str(batch_args.split_by)+' jobs.')
+totalchunks = 0
+for istation,current_station in all_stations_select.iterrows():
+    records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+    chunks_current_station = math.ceil(float(len(records_morning_station_select))/float(batch_args.split_by))
+    totalchunks +=chunks_current_station
+
+print('total chunks of simulations (= size of array-job) per experiment: ' + str(totalchunks))
+
+experiments = batch_args.experiments.strip(' ').split(' ')
+if batch_args.experiments_names is not None:
+    experiments_names = batch_args.experiments_names.strip(' ').split(' ')
+    if len(experiments_names) != len(experiments):
+        raise ValueError('Lenght of --experiments_names is different from --experiments')
+else:
+    experiments_names = experiments
+
+odir_exists = False
+
+cleanup = (batch_args.cleanup_output_directories == 'True')
+
+if not cleanup:
+    for expname in experiments_names:
+        if os.path.exists(batch_args.path_experiments+'/'+expname):
+            print("Output directory already exists: "+batch_args.path_experiments+'/'+expname+". ")
+            odir_exists = True
+if odir_exists:
+    raise IOError("At least one of the output directories exists. Please use '--cleanup_output_directories True' to delete any output directory.")
+else:
+    for iexp,expname in enumerate(experiments_names):
+        if cleanup:
+            if os.path.exists(batch_args.path_experiments+'/'+expname):
+                print("Warning! Output directory '"+batch_args.path_experiments+'/'+expname+"' exists! I'm removing it in 10 seconds!' Press ctrl-c to abort.")
+                sleep(10)
+                os.system("rm -R "+batch_args.path_experiments+'/'+expname)
+        if batch_args.multi_processing_mode == 'qsub':
+    
+            # C4GLJOB_timestamp="+dt.datetime.now().isoformat()+",
+            command = 'qsub '+batch_args.pbs_string+' '+batch_args.c4gl_path_lib+'/simulations/batch_simulations.pbs -t 0-'+\
+                        str(totalchunks-1)+" -v C4GLJOB_experiments="+str(experiments[iexp])+",C4GLJOB_experiments_names="+str(expname)
+            # propagate arguments towards the job script
+            for argkey in batch_args.__dict__.keys():
+                if ((argkey not in ['multi_processing_mode','cpu_count','experiments','experiments_names','pbs_string','cleanup_output_directories']) and \
+                    # default values are specified in the simulation script, so
+                    # excluded here
+                    (batch_args.__dict__[argkey] is not None)
+                   ):
+                        command +=',C4GLJOB_'+argkey+'='+str(batch_args.__dict__[argkey])
+    
+            print('Submitting array job for experiment '+expname+': '+command)
+            os.system(command)
+
+        elif batch_args.multi_processing_mode == 'pythonpool':
+            from multiprocessing import Pool                                       
+            
+            # # load moodule from absolute path
+            # https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
+            import importlib.util
+            print(batch_args.exec)
+            spec = importlib.util.spec_from_file_location("module.name", batch_args.exec)
+            task_module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(task_module)
+            print('hello')
+            print(batch_args.exec)
+            
+
+            args_dict_current = {**batch_args.__dict__}
+
+            # we avoid to pass Nones, so that defaults are taken from the child
+            # script
+            for removekey,nonevalue in batch_args.__dict__.items():
+                if nonevalue is None:
+                    args_dict_current.pop(removekey)
+
+            # remove keys that are not relevant in the child script, so not
+            # passed (or those that are redefined in the host script manually)
+            for key in ['exec','multi_processing_mode','cpu_count','experiments','experiments_names','pbs_string','cleanup_output_directories']:
+                if key in args_dict_current:
+                    args_dict_current.pop(key)
+
+            args_dict_current['experiments'] = experiments[iexp]
+            args_dict_current['experiments_names'] = expname
+
+            print(args_dict_current)
+            all_tasks = []
+            for ichunk in range(totalchunks):
+                all_tasks.append({'global_chunk_number':str(ichunk),**args_dict_current}) 
+
+            print(pd.DataFrame(all_tasks)) 
+            def parallelize(analysis, filenames, processes):
+                '''
+                Call `analysis` for each file in the sequence `filenames`, using
+                up to `processes` parallel processes. Wait for them all to complete
+                and then return a list of results.
+                '''
+                return Pool(processes).map(analysis, filenames, chunksize = 1)
+    
+            def execute_kwargs(x):
+                return task_module.execute(**x)
+
+            parallelize(execute_kwargs,all_tasks,int(batch_args.cpu_count))
+
+    #os.system(command)
+# elif sys.argv[1] == 'wsub':
+#     
+#     # with wsub
+#     STNlist = list(df_stations.iterrows())
+#     NUMSTNS = len(STNlist)
+#     PROCS = NUMSTNS 
+#     BATCHSIZE = 1 #math.ceil(np.float(NUMSTNS)/np.float(PROCS))
+# 
+#     os.system('wsub -batch /user/data/gent/gvo000/gvo00090/D2D/scripts/C4GL/global_run.pbs -t 0-'+str(PROCS-1))
+
diff --git a/class4gl/simulations/copy_update.py b/class4gl/simulations/copy_update.py
new file mode 100644
index 0000000..defa8c3
--- /dev/null
+++ b/class4gl/simulations/copy_update.py
@@ -0,0 +1,332 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+
+arguments = []
+
+#parser.add_argument('--timestamp')
+arguments.append(dict(arg='--path_forcing',\
+                    help='directory of forcing data to initialize and constrain the ABL model simulations'))
+arguments.append(dict(arg='--path_output',
+                    help='output directory in which the output as subdirectories are stored'))#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+arguments.append(dict(arg='--first_station_row',\
+                    help='starting row number of stations table'))
+arguments.append(dict(arg='--last_station_row',\
+                    help='ending row number of stations table'))
+arguments.append(dict(arg='--global_vars',\
+                    help="global vars to update ( ':'-seperated list) "))
+arguments.append(dict(arg='--station_id',\
+                    help="process a specific station id"))
+arguments.append(dict(arg='--error_handling',\
+                    default='dump_on_success',\
+                    help="type of error handling: either\n - 'dump_on_success' (default)\n - 'dump_always'"))
+arguments.append(dict(arg='--diag_tropo',\
+                    default=['advt','advq','advu','advv'],\
+                    help="field to diagnose the mean in the troposphere (<= 3000m)"))
+arguments.append(dict(arg='--subset_forcing',
+                    default='ini', 
+                    help="This indicates which yaml subset to initialize with.  Most common options are 'ini' (default) and 'morning'."))
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+
+arguments.append(dict(arg='--split_by',\
+                    type=int,
+                    help="the maxmimum number of soundings that are contained in each output file of a station. -1 means unlimited (default). In case of arrays experiments, this is usually overwritten by 50."))
+
+#arguments.append(dict(arg='--station-chunk',default=0)
+arguments.append(dict(arg='--c4gl_path_lib',help="the path of the CLASS4GL program"))#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+arguments.append(dict(arg='--global_chunk_number',help="this is the batch number of the expected series of experiments according to split_by"))
+arguments.append(dict(arg='--station_chunk_number',help="this is the batch number according to split_by in case of considering one station"))
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    #parser.add_argument('--timestamp')
+    for argument in arguments:
+        name = argument.pop('arg')
+        parser.add_argument(name,**argument)
+
+    args = parser.parse_args()
+else:
+    class Namespace:
+        def __init__(self,**kwargs):
+            self.__dict__.update(kwargs)
+
+    args = Namespace()
+    for argument in arguments:
+        if 'default' in argument.keys():
+            args.__dict__[argument['arg'].strip('-')] = argument['default']
+        else:
+            args.__dict__[argument['arg'].strip('-')] = None
+    print(args.__dict__)
+        
+
+def execute(**kwargs):
+    # note that with args, we actually mean the same as those specified with
+    # the argparse module above
+    
+    # overwrite the args according to the kwargs when the procedure is called
+    # as module function
+    for key,value in kwargs.items():
+        args.__dict__[key]  = value
+    
+    print("-- begin arguments --")
+    for key,value in args.__dict__.items():
+         print(key,': ',value)
+    print("-- end arguments ----")
+    
+    # load specified class4gl library
+    if args.c4gl_path_lib is not None:
+        sys.path.insert(0, args.c4gl_path_lib)
+    
+    from class4gl import class4gl_input, data_global,class4gl
+    from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+    from class4gl import blh,class4gl_input
+    
+    # this is a variant of global run in which the output of runs are still written
+    # out even when the run crashes.
+    
+    # #only include the following timeseries in the model output
+    # timeseries_only = \
+    # ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+    #  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+    #  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+    #  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+    #  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+    
+    if (args.global_vars is not None):
+        globaldata = data_global()
+        globaldata.load_datasets(recalc=0)
+    
+    
+    # ========================
+    print("getting a list of stations")
+    # ========================
+    
+    # these are all the stations that are found in the input dataset
+    all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+    
+    # ====================================
+    print('defining all_stations_select')
+    # ====================================
+    
+    # these are all the stations that are supposed to run by the whole batch (all
+    # chunks). We narrow it down according to the station(s) specified.
+    
+    if args.station_id is not None:
+        print("Selecting station by ID")
+        stations_iter = stations_iterator(all_stations)
+        STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+        all_stations_select = pd.DataFrame([run_station])
+    else:
+        print("Selecting stations from a row range in the table")
+        all_stations_select = pd.DataFrame(all_stations.table)
+        if args.last_station_row is not None:
+            all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+        if args.first_station_row is not None:
+            all_stations_select = all_station_select.iloc[int(args.first_station):]
+    print("station numbers included in the whole batch "+\
+          "(all chunks):",list(all_stations_select.index))
+    
+    print(all_stations_select)
+    print("getting all records of the whole batch")
+    all_records_morning_select = get_records(all_stations_select,\
+                                             args.path_forcing,\
+                                             subset=args.subset_forcing,
+                                             refetch_records=False,
+                                             )
+    
+    # only run a specific chunck from the selection
+    if args.global_chunk_number is not None:
+        if args.station_chunk_number is not None:
+            raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+    
+        if (args.split_by is None) or (args.split_by <= 0):
+                raise ValueError("global_chunk_number is specified, but --split_by is not a strict positive number, so I don't know how to split the batch into chunks.")
+    
+        run_station_chunk = None
+        print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+        totalchunks = 0
+        stations_iter = all_stations_select.iterrows()
+        in_current_chunk = False
+        try:
+            while not in_current_chunk:
+                istation,current_station = stations_iter.__next__()
+                all_records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+                chunks_current_station = math.ceil(float(len(all_records_morning_station_select))/float(args.split_by))
+                print('chunks_current_station',chunks_current_station)
+                in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+            
+                if in_current_chunk:
+                    run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                    run_station_chunk = int(args.global_chunk_number) - totalchunks 
+            
+                totalchunks +=chunks_current_station
+            
+    
+        except StopIteration:
+           raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+        print("station = ",list(run_stations.index))
+        print("station chunk number:",run_station_chunk)
+    
+    # if no global chunk is specified, then run the whole station selection in one run, or
+    # a specific chunk for each selected station according to # args.station_chunk_number
+    else:
+        run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+        if args.station_chunk_number is not None:
+            run_station_chunk = int(args.station_chunk_number)
+            print("station(s) that is processed.",list(run_stations.index))
+            print("chunk number: ",run_station_chunk)
+        else:
+            if args.split_by is not None:
+                raise ValueError("Chunks are defined by --split_by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split_by.")
+            run_station_chunk = 0
+            print("stations that are processed.",list(run_stations.index))
+            
+    
+    #print(all_stations)
+    print('Fetching initial/forcing records')
+    records_morning = get_records(run_stations,\
+                                  args.path_forcing,\
+                                  subset=args.subset_forcing,
+                                  refetch_records=False,
+                                  )
+    if len(records_morning) == 0:
+        raise IOError("No initialization records records found in "+\
+                      args.path_forcing+' (subset: '+args_subset_forcing+')')
+    
+    # note that if runtime is an integer number, we don't need to get the afternoon
+    # profiles. 
+    
+
+    
+    path_output = args.path_output
+    
+    os.system('mkdir -p '+path_output)
+    for istation,current_station in run_stations.iterrows():
+        # records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+        records_morning_station = records_morning.loc[(current_station.name):(current_station.name)]
+
+        fn_morning = args.path_forcing+'/'+format(current_station.name,'05d')+'_'+args.subset_forcing+'.yaml'
+        if os.path.isfile(fn_morning):
+            file_morning = open(fn_morning)
+        else:
+            fn_morning = \
+                 args.path_forcing+'/'+format(current_station.name,'05d')+\
+                 '_'+str(run_station_chunk)+'_'+args.subset_forcing+'.yaml'
+            file_morning = open(fn_morning)
+    
+        # if args.runtime == 'from_profile_pair':
+        #     file_afternoon = open(args.path_forcing+'/'+format(current_station.name,'05d')+'_end.yaml')
+        fn_ini = path_output+'/'+format(current_station.name,'05d')+'_'+\
+                 str(int(run_station_chunk))+'_ini.yaml'
+        file_ini = open(fn_ini,'w')
+    
+        #iexp = 0
+        onerun = False
+        print('starting station chunk number: '\
+              +str(run_station_chunk)+' (chunk size:',args.split_by,')')
+    
+        skip_chunk = False
+        if 'chunk' in records_morning.index.names:
+            records_morning_station_chunk = records_morning_station.loc[(current_station.name,run_station_chunk):(current_station.name,run_station_chunk)]
+        else:
+            start_record = run_station_chunk*args.split_by if run_station_chunk is not 0 else 0
+            end_record = (run_station_chunk+1)*args.split_by if args.split_by is not None else None
+            if start_record >= (len(records_morning_station)):
+                print("warning: outside of profile number range for station "+\
+                      str(current_station)+". Skipping chunk number for this station.")
+                skip_chunk = True
+                records_morning_station_chunk = None
+            else:
+                records_morning_station_chunk = records_morning_station.iloc[start_record:end_record] #  [(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+
+        if not skip_chunk:
+    
+            isim = 0
+            for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                    print('starting '+str(isim+1)+' out of '+\
+                      str(len(records_morning_station_chunk) )+\
+                      ' (station total: ',str(len(records_morning_station)),')')  
+                
+            
+                    c4gli_morning = get_record_yaml(file_morning, 
+                                                    record_morning.index_start, 
+                                                    record_morning.index_end,
+                                                    mode='model_input')
+                    
+                    
+    
+            
+                    if args.global_vars is not None:
+                        c4gli_morning.get_global_input(globaldata,only_keys=args.global_vars.strip().split(':'))
+    
+                    onerun = True
+    
+                    print("dumping to "+str(file_ini)+ ' ('+fn_ini+')') 
+                    c4gli_morning.dump(file_ini)
+                        
+                        
+                    isim += 1
+    
+    
+            file_ini.close()
+            file_morning.close()
+    
+            if onerun:
+                records_ini = get_records(pd.DataFrame([current_station]),\
+                                                           path_output,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='ini',
+                                                           refetch_records=True,
+                                                           )
+            else:
+                # remove empty files
+                os.system('rm '+fn_ini)
+    
+    # # align afternoon records with initial records, and set same index
+    # records_afternoon.index = records_afternoon.ldatetime.dt.date
+    # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+    # records_afternoon.index = records_ini.index
+    
+    # stations_for_iter = stations(path_output)
+    # for STNID,station in stations_iterator(stations_for_iter):
+    #     records_current_station_index = \
+    #             (records_ini.index.get_level_values('STNID') == STNID)
+    #     file_current_station_end_mod = STNID
+    # 
+    #     with \
+    #     open(path_output+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    #     open(path_output+'/'+format(STNID,"05d")+'_end_mod.yaml','r') as file_station_end_mod, \
+    #     open(path_forcing+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+    #         for (STNID,index),record_ini in records_iterator(records_ini):
+    #             c4gli_ini = get_record_yaml(file_station_ini, 
+    #                                         record_ini.index_start, 
+    #                                         record_ini.index_end,
+    #                                         mode='ini')
+    #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+    # 
+    #             record_end_mod = records_end_mod.loc[(STNID,index)]
+    #             c4gl_end_mod = get_record_yaml(file_station_end_mod, 
+    #                                         record_end_mod.index_start, 
+    #                                         record_end_mod.index_end,
+    #                                         mode='mod')
+    #             record_afternoon = records_afternoon.loc[(STNID,index)]
+    #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+    #                                         record_afternoon.index_start, 
+    #                                         record_afternoon.index_end,
+    #                                         mode='ini')
+
+
+if __name__ == '__main__':
+    #execute(**vars(args))
+    execute()
diff --git a/runmodel.py b/class4gl/simulations/runmodel.py
similarity index 100%
rename from runmodel.py
rename to class4gl/simulations/runmodel.py
diff --git a/class4gl/simulations/simulations.py b/class4gl/simulations/simulations.py
new file mode 100644
index 0000000..08ec611
--- /dev/null
+++ b/class4gl/simulations/simulations.py
@@ -0,0 +1,476 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+import logging
+
+
+arguments = []
+
+#parser.add_argument('--timestamp')
+arguments.append(dict(arg='--path_forcing',\
+                    help='directory of forcing data to initialize and constrain the ABL model simulations'))
+arguments.append(dict(arg='--path_experiments',
+                    help='output directory in which the experiments as subdirectories are stored'))#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+arguments.append(dict(arg='--first_station_row',\
+                    help='starting row number of stations table'))
+arguments.append(dict(arg='--last_station_row',\
+                    help='ending row number of stations table'))
+arguments.append(dict(arg='--station_id',\
+                    help="process a specific station id"))
+arguments.append(dict(arg='--error_handling',\
+                    default='dump_on_success',\
+                    help="type of error handling: either\n - 'dump_on_success' (default)\n - 'dump_always'"))
+arguments.append(dict(arg='--diag_tropo',\
+                    default='',#\'advt,advq,advu,advv',\
+                    help="field to diagnose the mean in the troposphere (<= 3000m)"))
+arguments.append(dict(arg='--subset_forcing',
+                    default='ini', 
+                    help="This indicates which yaml subset to initialize with.  Most common options are 'ini' (default) and 'morning'."))
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+arguments.append(dict(arg='--runtime',
+                    default='from_input',
+                    help="set the runtime of the simulation in seconds, or get it from the daytime difference in the profile pairs 'from_input' (default)"))
+
+arguments.append(dict(arg='--experiments', help="IDs of experiments, as a space-seperated list (default: 'BASE')"))
+arguments.append(dict(arg='--split_by',\
+                    type=int,
+                    help="the maxmimum number of soundings that are contained in each output file of a station. -1 means unlimited (default). In case of arrays experiments, this is usually overwritten by 50."))
+
+#arguments.append(dict(arg='--station-chunk',default=0)
+arguments.append(dict(arg='--c4gl_path_lib',help="the path of the CLASS4GL program"))#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+arguments.append(dict(arg='--global_chunk_number',help="this is the batch number of the expected series of experiments according to split_by"))
+arguments.append(dict(arg='--station_chunk_number',help="this is the batch number according to split_by in case of considering one station"))
+arguments.append(dict(arg='--experiments_names', help="Alternative output names that are given to the experiments. By default, these are the same as --experiments") )
+arguments.append(dict(arg='--debug_level', type=str,default=None,help="Debug level according to the standard python logging module. ") )
+# {'CRITICAL': 50,
+#   'FATAL': 50,
+#   'ERROR': 40,
+#   'WARN': 30,
+#   'WARNING': 30,
+#   'INFO': 20,
+#   'DEBUG': 10,
+#   'NOTSET': 0}
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    #parser.add_argument('--timestamp')
+    for argument in arguments:
+        name = argument.pop('arg')
+        parser.add_argument(name,**argument)
+
+    args = parser.parse_args()
+else:
+    class Namespace:
+        def __init__(self,**kwargs):
+            self.__dict__.update(kwargs)
+
+    args = Namespace()
+    for argument in arguments:
+        if 'default' in argument.keys():
+            args.__dict__[argument['arg'].strip('-')] = argument['default']
+        else:
+            args.__dict__[argument['arg'].strip('-')] = None
+    print(args.__dict__)
+        
+
+def execute(**kwargs):
+    # note that with args, we actually mean the same as those specified with
+    # the argparse module above
+    
+    # overwrite the args according to the kwargs when the procedure is called
+    # as module function
+    for key,value in kwargs.items():
+        args.__dict__[key]  = value
+    
+    print("-- begin arguments --")
+    for key,value in args.__dict__.items():
+         print(key,': ',value)
+    print("-- end arguments ----")
+    
+    # load specified class4gl library
+    if args.c4gl_path_lib is not None:
+        sys.path.insert(0, args.c4gl_path_lib)
+    
+    from class4gl import class4gl_input, data_global,class4gl
+    from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+    from class4gl import blh,class4gl_input
+    
+    # this is a variant of global run in which the output of runs are still written
+    # out even when the run crashes.
+    
+    # #only include the following timeseries in the model output
+    # timeseries_only = \
+    # ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+    #  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+    #  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+    #  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+    #  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+    
+    
+    # for iEXP in range(4):
+    #     EXPKEY = 'LCZ'+str(iEXP)
+    #     EXP_DEFS[EXPKEY] = {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'urban':'LCZ'+str(iEXP)}
+
+
+    EXP_DEFS  =\
+    {
+      'LCZ':{'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'urban':'lcw1'},
+
+      'BASE':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+    
+      'NOADV':{'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+        
+      'ERA_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+      'NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+      'ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+      'W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+      'AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+      'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+      'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+      'GLOBAL_ADV_SM2':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+      'IOPS_ADV_SM2':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+      'GLOBAL_ADV_ERA_NEW':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+        'GLOBAL_ADV_SHR':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False,'sw_shearwe':True},
+      'GLOBAL_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+      'GLOBAL_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+      'IOPS_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+      'IOPS_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+      'IOPS_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+      'IOPS_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+    }
+    
+    # ========================
+    print("getting a list of stations")
+    # ========================
+    
+    # these are all the stations that are found in the input dataset
+    all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+    
+    # ====================================
+    print('defining all_stations_select')
+    # ====================================
+    
+    # these are all the stations that are supposed to run by the whole batch (all
+    # chunks). We narrow it down according to the station(s) specified.
+    
+    
+    
+    if args.station_id is not None:
+        print("Selecting station by ID")
+        stations_iter = stations_iterator(all_stations)
+        STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+        all_stations_select = pd.DataFrame([run_station])
+    else:
+        print("Selecting stations from a row range in the table")
+        all_stations_select = pd.DataFrame(all_stations.table)
+        if args.last_station_row is not None:
+            all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+        if args.first_station_row is not None:
+            all_stations_select = all_station_select.iloc[int(args.first_station):]
+    print("station numbers included in the whole batch "+\
+          "(all chunks):",list(all_stations_select.index))
+    
+    print(all_stations_select)
+    print("getting all records of the whole batch")
+    all_records_morning_select = get_records(all_stations_select,\
+                                             args.path_forcing,\
+                                             subset=args.subset_forcing,
+                                             refetch_records=False,
+                                             )
+    
+    # only run a specific chunck from the selection
+    if args.global_chunk_number is not None:
+        if args.station_chunk_number is not None:
+            raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+    
+        if (args.split_by is None) or (args.split_by <= 0):
+                raise ValueError("global_chunk_number is specified, but --split_by is not a strict positive number, so I don't know how to split the batch into chunks.")
+    
+        run_station_chunk = None
+        print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+        totalchunks = 0
+        stations_iter = all_stations_select.iterrows()
+        in_current_chunk = False
+        try:
+            while not in_current_chunk:
+                istation,current_station = stations_iter.__next__()
+                all_records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+                chunks_current_station = math.ceil(float(len(all_records_morning_station_select))/float(args.split_by))
+                print('chunks_current_station',chunks_current_station)
+                in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+            
+                if in_current_chunk:
+                    run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                    run_station_chunk = int(args.global_chunk_number) - totalchunks 
+            
+                totalchunks +=chunks_current_station
+            
+    
+        except StopIteration:
+           raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+        print("station = ",list(run_stations.index))
+        print("station chunk number:",run_station_chunk)
+    
+    # if no global chunk is specified, then run the whole station selection in one run, or
+    # a specific chunk for each selected station according to # args.station_chunk_number
+    else:
+        run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+        if args.station_chunk_number is not None:
+            run_station_chunk = int(args.station_chunk_number)
+            print("station(s) that is processed.",list(run_stations.index))
+            print("chunk number: ",run_station_chunk)
+        else:
+            if args.split_by is not None:
+                raise ValueError("Chunks are defined by --split_by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split_by.")
+            run_station_chunk = 0
+            print("stations that are processed.",list(run_stations.index))
+            
+    
+    #print(all_stations)
+    print('Fetching initial/forcing records')
+    records_morning = get_records(run_stations,\
+                                  args.path_forcing,\
+                                  subset=args.subset_forcing,
+                                  refetch_records=False,
+                                  )
+    if len(records_morning) == 0:
+        raise IOError("No initialization records records found in "+\
+                      args.path_forcing+' (subset: '+args_subset_forcing+')')
+    
+    # note that if runtime is an integer number, we don't need to get the afternoon
+    # profiles. 
+    if args.runtime == 'from_profile_pair':
+        print('Fetching afternoon records for determining the simulation runtimes')
+        records_afternoon = get_records(run_stations,\
+                                        args.path_forcing,\
+                                        subset='end',
+                                        refetch_records=False,
+                                        )
+        if len(records_afternoon) == 0:
+            raise IOError("No final state records found in "+\
+                          args.path_forcing+' (subset: '+args_subset_forcing+')')
+        
+        # print(records_morning.index)
+        # print(records_afternoon.index)
+        # align afternoon records with the noon records, and set same index
+        print('hello')
+        print(len(records_afternoon))
+        print(len(records_morning))
+    
+        print("aligning morning and afternoon records")
+        records_morning['dates'] = records_morning['ldatetime'].dt.date
+        records_afternoon['dates'] = records_afternoon['ldatetime'].dt.date
+        records_afternoon.set_index(['STNID','dates'],inplace=True)
+        ini_index_dates = records_morning.set_index(['STNID','dates']).index
+        records_afternoon = records_afternoon.loc[ini_index_dates]
+        records_afternoon.index = records_morning.index
+    
+    experiments = args.experiments.strip(' ').split(' ')
+    if args.experiments_names is not None:
+        experiments_names = args.experiments_names.strip(' ').split(' ')
+        if len(experiments_names) != len(experiments):
+            raise ValueError('Lenght of --experiments_names is different from --experiments')
+    
+    else:
+        experiments_names = experiments
+    
+    for iexpname,expid in enumerate(experiments):
+        expname = experiments_names[iexpname]
+        exp = EXP_DEFS[expid]
+        path_exp = args.path_experiments+'/'+expname+'/'
+    
+        os.system('mkdir -p '+path_exp)
+        for istation,current_station in run_stations.iterrows():
+            print(istation,current_station)
+            records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+            start_record = run_station_chunk*args.split_by if run_station_chunk is not 0 else 0
+            end_record = (run_station_chunk+1)*args.split_by if args.split_by is not None else None
+            if start_record >= (len(records_morning_station)):
+                print("warning: outside of profile number range for station "+\
+                      str(current_station)+". Skipping chunk number for this station.")
+            else:
+                fn_morning = args.path_forcing+'/'+format(current_station.name,'05d')+'_'+args.subset_forcing+'.yaml'
+                if os.path.isfile(fn_morning):
+                    file_morning = open(fn_morning)
+                else:
+                    fn_morning = \
+                         args.path_forcing+'/'+format(current_station.name,'05d')+\
+                         '_'+str(run_station_chunk)+'_'+args.subset_forcing+'.yaml'
+                    file_morning = open(fn_morning)
+    
+                if args.runtime == 'from_profile_pair':
+                    file_afternoon = open(args.path_forcing+'/'+format(current_station.name,'05d')+'_end.yaml')
+                fn_ini = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                         str(int(run_station_chunk))+'_ini.yaml'
+                fn_end_mod = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                         str(int(run_station_chunk))+'_end.yaml'
+                file_ini = open(fn_ini,'w')
+                file_end_mod = open(fn_end_mod,'w')
+    
+                #iexp = 0
+                onerun = False
+                print('starting station chunk number: '\
+                      +str(run_station_chunk)+' (chunk size:',args.split_by,')')
+    
+                records_morning_station_chunk = records_morning_station.iloc[start_record:end_record] #  [(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+    
+                isim = 0
+                for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                        print('starting '+str(isim+1)+' out of '+\
+                          str(len(records_morning_station_chunk) )+\
+                          ' (station total: ',str(len(records_morning_station)),')')  
+                    
+                
+                        c4gli_morning = get_record_yaml(file_morning, 
+                                                        record_morning.index_start, 
+                                                        record_morning.index_end,
+                                                        mode='model_input')
+                        if args.diag_tropo is not '':
+                            print('add tropospheric parameters on advection and subsidence (for diagnosis)')
+                            seltropo = (c4gli_morning.air_ac.p > c4gli_morning.air_ac.p.iloc[-1]+ 3000.*(- 1.2 * 9.81 ))
+                            profile_tropo = c4gli_morning.air_ac[seltropo]
+                            for var in args.diag_tropo.split(','):#['t','q','u','v',]:
+                                if var[:3] == 'adv':
+                                    mean_adv_tropo = np.mean(profile_tropo[var+'_x']+profile_tropo[var+'_y'] )
+                                    c4gli_morning.update(source='era-interim',pars={var+'_tropo':mean_adv_tropo})
+                                else:
+                                    print("warning: tropospheric variable "+var+" not recognized")
+                        
+                        
+                        if args.runtime == 'from_profile_pair':
+                            record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+                            c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                                              int(record_afternoon.index_start),
+                                                              int(record_afternoon.index_end),
+                                                            mode='model_input')
+                            runtime = int((c4gli_afternoon.pars.datetime_daylight - 
+                                                 c4gli_morning.pars.datetime_daylight).total_seconds())
+                        elif args.runtime == 'from_input':
+                            runtime = c4gli_morning.pars.runtime
+                        else:
+                            runtime = int(args.runtime)
+    
+                
+                        c4gli_morning.update(source='pairs',pars={'runtime' : \
+                                            runtime})
+                        c4gli_morning.update(source=expname, pars=exp)
+    
+                        c4gl = class4gl(c4gli_morning,debug_level=args.debug_level)
+    
+                        if args.error_handling == 'dump_always':
+                            try:
+                                print('checking data sources')
+                                if not c4gli_morning.check_source_globaldata():
+                                    print('Warning: some input sources appear invalid')
+                                c4gl.run()
+                                print('run succesful')
+                            except Exception as inst:
+                                print('run not succesful. Error message is:')
+                                print(inst.args) 
+                            onerun = True
+    
+                            print("dumping to "+str(file_ini)+ ' ('+fn_ini+')') 
+                            c4gli_morning.dump(file_ini)
+                            
+                            
+                            c4gl.dump(file_end_mod,\
+                                      include_input=False,\
+                                      #timeseries_only=timeseries_only,\
+                                     )
+                            onerun = True
+                        # in this case, only the file will dumped if the runs were
+                        # successful
+                        elif args.error_handling == 'dump_on_success':
+                           try:
+                                print('checking data sources')
+                                if not c4gli_morning.check_source_globaldata():
+                                    print('Warning: some input sources appear invalid')
+                                c4gl.run()
+                                print('run succesful')
+                                c4gli_morning.dump(file_ini)
+                                
+                                
+                                print("dumping to "+str(file_ini)) 
+                                c4gl.dump(file_end_mod,\
+                                          include_input=False,\
+                                          #timeseries_only=timeseries_only,\
+                                         )
+                                onerun = True
+                           except Exception as inst:
+                               print('run not succesful. Error message is:')
+                               print(inst.args) 
+                               print('run not succesful')
+                        isim += 1
+    
+    
+                file_ini.close()
+                file_end_mod.close()
+                file_morning.close()
+                if args.runtime == 'from_profile_pair':
+                    file_afternoon.close()
+        
+                if onerun:
+                    records_ini = get_records(pd.DataFrame([current_station]),\
+                                                               path_exp,\
+                                                               getchunk = int(run_station_chunk),\
+                                                               subset='ini',
+                                                               refetch_records=True,
+                                                               )
+                    records_end_mod = get_records(pd.DataFrame([current_station]),\
+                                                               path_exp,\
+                                                               getchunk = int(run_station_chunk),\
+                                                               subset='end',\
+                                                               refetch_records=True,\
+                                                               )
+                else:
+                    # remove empty files
+                    os.system('rm '+fn_ini)
+                    os.system('rm '+fn_end_mod)
+        
+        # # align afternoon records with initial records, and set same index
+        # records_afternoon.index = records_afternoon.ldatetime.dt.date
+        # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+        # records_afternoon.index = records_ini.index
+        
+        # stations_for_iter = stations(path_exp)
+        # for STNID,station in stations_iterator(stations_for_iter):
+        #     records_current_station_index = \
+        #             (records_ini.index.get_level_values('STNID') == STNID)
+        #     file_current_station_end_mod = STNID
+        # 
+        #     with \
+        #     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+        #     open(path_exp+'/'+format(STNID,"05d")+'_end_mod.yaml','r') as file_station_end_mod, \
+        #     open(path_forcing+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+        #         for (STNID,index),record_ini in records_iterator(records_ini):
+        #             c4gli_ini = get_record_yaml(file_station_ini, 
+        #                                         record_ini.index_start, 
+        #                                         record_ini.index_end,
+        #                                         mode='ini')
+        #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+        # 
+        #             record_end_mod = records_end_mod.loc[(STNID,index)]
+        #             c4gl_end_mod = get_record_yaml(file_station_end_mod, 
+        #                                         record_end_mod.index_start, 
+        #                                         record_end_mod.index_end,
+        #                                         mode='mod')
+        #             record_afternoon = records_afternoon.loc[(STNID,index)]
+        #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+        #                                         record_afternoon.index_start, 
+        #                                         record_afternoon.index_end,
+        #                                         mode='ini')
+
+
+if __name__ == '__main__':
+    #execute(**vars(args))
+    execute()
diff --git a/class4gl/simulations/simulations_iter.py b/class4gl/simulations/simulations_iter.py
new file mode 100644
index 0000000..1270145
--- /dev/null
+++ b/class4gl/simulations/simulations_iter.py
@@ -0,0 +1,567 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+
+arguments = []
+
+#parser.add_argument('--timestamp')
+arguments.append(dict(arg='--path_forcing',\
+                    help='directory of forcing data to initialize and constrain the ABL model simulations'))
+arguments.append(dict(arg='--path_experiments',
+                    help='output directory in which the experiments as subdirectories are stored'))#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+arguments.append(dict(arg='--first_station_row',\
+                    help='starting row number of stations table'))
+arguments.append(dict(arg='--last_station_row',\
+                    help='ending row number of stations table'))
+arguments.append(dict(arg='--station_id',\
+                    help="process a specific station id"))
+arguments.append(dict(arg='--error_handling',\
+                    default='dump_on_success',\
+                    help="type of error handling: either\n - 'dump_on_success' (default)\n - 'dump_always'"))
+arguments.append(dict(arg='--diag_tropo',\
+                    default=['advt','advq','advu','advv'],\
+                    help="field to diagnose the mean in the troposphere (<= 3000m)"))
+arguments.append(dict(arg='--subset_forcing',
+                    default='ini', 
+                    help="This indicates which yaml subset to initialize with.  Most common options are 'ini' (default) and 'morning'."))
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+arguments.append(dict(arg='--runtime',
+                    default='from_input',
+                    help="set the runtime of the simulation in seconds, or get it from the daytime difference in the profile pairs 'from_input' (default)"))
+
+arguments.append(dict(arg='--experiments', help="IDs of experiments, as a space-seperated list (default: 'BASE')"))
+arguments.append(dict(arg='--split_by',\
+                    type=int,
+                    help="the maxmimum number of soundings that are contained in each output file of a station. -1 means unlimited (default). In case of arrays experiments, this is usually overwritten by 50."))
+
+#arguments.append(dict(arg='--station-chunk',default=0)
+arguments.append(dict(arg='--c4gl_path_lib',help="the path of the CLASS4GL program"))#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+arguments.append(dict(arg='--global_chunk_number',help="this is the batch number of the expected series of experiments according to split_by"))
+arguments.append(dict(arg='--station_chunk_number',help="this is the batch number according to split_by in case of considering one station"))
+arguments.append(dict(arg='--experiments_names', help="Alternative output names that are given to the experiments. By default, these are the same as --experiments") )
+
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    #parser.add_argument('--timestamp')
+    for argument in arguments:
+        name = argument.pop('arg')
+        parser.add_argument(name,**argument)
+
+    args = parser.parse_args()
+else:
+    class Namespace:
+        def __init__(self,**kwargs):
+            self.__dict__.update(kwargs)
+
+    args = Namespace()
+    for argument in arguments:
+        if 'default' in argument.keys():
+            args.__dict__[argument['arg'].strip('-')] = argument['default']
+        else:
+            args.__dict__[argument['arg'].strip('-')] = None
+    print(args.__dict__)
+        
+
+def execute(**kwargs):
+    # note that with args, we actually mean the same as those specified with
+    # the argparse module above
+
+    # overwrite the args according to the kwargs when the procedure is called
+    # as module function
+    for key,value in kwargs.items():
+        args.__dict__[key]  = value
+
+    print("-- begin arguments --")
+    for key,value in args.__dict__.items():
+         print(key,': ',value)
+    print("-- end arguments ----")
+
+    # load specified class4gl library
+    if args.c4gl_path_lib is not None:
+        sys.path.insert(0, args.c4gl_path_lib)
+    
+    from class4gl import class4gl_input, data_global,class4gl
+    from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+    from class4gl import blh,class4gl_input
+    
+    # this is a variant of global run in which the output of runs are still written
+    # out even when the run crashes.
+    
+    # #only include the following timeseries in the model output
+    # timeseries_only = \
+    # ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+    #  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+    #  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+    #  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+    #  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+    
+    
+    EXP_DEFS  =\
+    {
+      'BASE_ITER':{'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+      'BASE_ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+      'BASE_ITER_W_ADV':{'sw_ac' : ['adv',"w"],'sw_ap': True,'sw_lit': False},
+      'BASE_ITER_W':{'sw_ac' : ["w"],'sw_ap': True,'sw_lit': False},
+        'BASE_ITER_ADV_B05':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False,'beta':0.2},
+    
+        
+      'ERA_NOAC_ITER':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+      'NOAC_ITER':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+      'ADV_ITER':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+      'W_ITER':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+      'AC_ITER': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+      'GLOBAL_NOAC_ITER':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+      'GLOBAL_ADV_ITER':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+    }
+    
+    
+    # #SET = 'GLOBAL'
+    # SET = args.dataset
+    
+    # ========================
+    print("getting a list of stations")
+    # ========================
+    
+    # these are all the stations that are found in the input dataset
+    all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+    
+    # ====================================
+    print('defining all_stations_select')
+    # ====================================
+    
+    # these are all the stations that are supposed to run by the whole batch (all
+    # chunks). We narrow it down according to the station(s) specified.
+    
+    
+    
+    if args.station_id is not None:
+        print("Selecting station by ID")
+        stations_iter = stations_iterator(all_stations)
+        STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+        all_stations_select = pd.DataFrame([run_station])
+    else:
+        print("Selecting stations from a row range in the table")
+        all_stations_select = pd.DataFrame(all_stations.table)
+        if args.last_station_row is not None:
+            all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+        if args.first_station_row is not None:
+            all_stations_select = all_station_select.iloc[int(args.first_station):]
+    print("station numbers included in the whole batch "+\
+          "(all chunks):",list(all_stations_select.index))
+    
+    print(all_stations_select)
+    print("getting all records of the whole batch")
+    all_records_morning_select = get_records(all_stations_select,\
+                                             args.path_forcing,\
+                                             subset=args.subset_forcing,
+                                             refetch_records=False,
+                                             )
+    
+    # only run a specific chunck from the selection
+    if args.global_chunk_number is not None:
+        if args.station_chunk_number is not None:
+            raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+    
+    
+        if not (int(args.split_by) > 0) :
+                raise ValueError("global_chunk_number is specified, but --split-by is not a strict positive number, so I don't know how to split the batch into chunks.")
+    
+        run_station_chunk = None
+        print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+        totalchunks = 0
+        stations_iter = all_stations_select.iterrows()
+        in_current_chunk = False
+        try:
+            while not in_current_chunk:
+                istation,current_station = stations_iter.__next__()
+                all_records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+                chunks_current_station = math.ceil(float(len(all_records_morning_station_select))/float(args.split_by))
+                print('chunks_current_station',chunks_current_station)
+                in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+            
+                if in_current_chunk:
+                    run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                    run_station_chunk = int(args.global_chunk_number) - totalchunks 
+            
+                totalchunks +=chunks_current_station
+            
+    
+        except StopIteration:
+           raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+        print("station = ",list(run_stations.index))
+        print("station chunk number:",run_station_chunk)
+    
+    # if no global chunk is specified, then run the whole station selection in one run, or
+    # a specific chunk for each selected station according to # args.station_chunk_number
+    else:
+        run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+        if args.station_chunk_number is not None:
+            run_station_chunk = int(args.station_chunk_number)
+            print("station(s) that is processed.",list(run_stations.index))
+            print("chunk number: ",run_station_chunk)
+        else:
+            if args.split_by != -1:
+                raise ValueError("Chunks are defined by --split_by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split-by.")
+            run_station_chunk = 0
+            print("stations that are processed.",list(run_stations.index))
+            
+    
+    #print(all_stations)
+    print('Fetching initial/forcing records')
+    records_morning = get_records(run_stations,\
+                                  args.path_forcing,\
+                                  subset=args.subset_forcing,
+                                  refetch_records=False,
+                                  )
+    if len(records_morning) == 0:
+        raise IOError("No initialization records records found in "+\
+                      args.path_forcing+' (subset: '+args_subset_forcing+')')
+
+
+    
+    # note that if runtime is an integer number, we don't need to get the afternoon
+    # profiles. 
+    if args.runtime == 'from_profile_pair':
+        print('Fetching afternoon records for determining the simulation runtimes')
+        records_afternoon = get_records(run_stations,\
+                                        args.path_forcing,\
+                                        subset='end',
+                                        refetch_records=False,
+                                        )
+        if len(records_afternoon) == 0:
+            raise IOError("No final state records found in "+\
+                          args.path_forcing+' (subset: '+args_subset_forcing+')')
+        
+        # print(records_morning.index)
+        # print(records_afternoon.index)
+        # align afternoon records with the noon records, and set same index
+        print(len(records_afternoon))
+        print(len(records_morning))
+    
+        print("aligning morning and afternoon records")
+        records_morning['dates'] = records_morning['ldatetime'].dt.date
+        records_afternoon['dates'] = records_afternoon['ldatetime'].dt.date
+        records_afternoon.set_index(['STNID','dates'],inplace=True)
+        ini_index_dates = records_morning.set_index(['STNID','dates']).index
+        records_afternoon = records_afternoon.loc[ini_index_dates]
+        records_afternoon.index = records_morning.index
+    
+
+
+    experiments = args.experiments.strip(' ').split(' ')
+    if args.experiments_names is not None:
+        experiments_names = args.experiments_names.strip(' ').split(' ')
+        if len(experiments_names) != len(experiments):
+            raise ValueError('Lenght of --experiments_names is different from --experiments')
+    
+    else:
+        experiments_names = experiments
+    
+    for iexpname,expid in enumerate(experiments):
+        expname = experiments_names[iexpname]
+        exp = EXP_DEFS[expid]
+        path_exp = args.path_experiments+'/'+expname+'/'
+    
+        os.system('mkdir -p '+path_exp)
+        records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+        # records_afternoon_station = records_afternoon.query('STNID == '+str(current_station.name))
+        # for varkey in ['h','theta']:
+        #     records_morning_station['d'+varkey+'dt'] = \
+        #             (records_afternoon_station[sourcekey][varkey] - records_morning_station[sourcekey][varkey])/\
+        #             (records_afternoon_station[sourcekey].ldatetime - records_morning_station[sourcekey].ldatetime).dt.seconds*3600.
+        # select_loc = records_morning_station.query( '(dthetadt > 0) & (dhdt > 0.)').index
+        # records_morning_station = records_morning_station.loc[select_loc]
+        # records_afternoon_station = records_afternoon_station.loc[select_loc]
+
+        for istation,current_station in run_stations.iterrows():
+            if (int(args.split_by) * int(run_station_chunk)) >= (len(records_morning_station)):
+                print("warning: outside of profile number range for station "+\
+                      str(current_station)+". Skipping chunk number for this station.")
+            else:
+    
+                fn_morning = args.path_forcing+'/'+format(current_station.name,'05d')+'_'+args.subset_forcing+'.yaml'
+                print('fn_morning',fn_morning)
+                if os.path.isfile(fn_morning):
+                    file_morning = open(fn_morning)
+                else:
+                    fn_morning = \
+                         args.path_forcing+'/'+format(current_station.name,'05d')+\
+                         '_'+str(run_station_chunk)+'_'+args.subset_forcing+'.yaml'
+                    file_morning = open(fn_morning)
+    
+                fn_afternoon = args.path_forcing+'/'+format(current_station.name,'05d')+'_end.yaml'
+                print(fn_afternoon)
+                if args.runtime == 'from_profile_pair':
+                    file_afternoon = open(fn_afternoon)
+                fn_ini = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                         str(int(run_station_chunk))+'_ini.yaml'
+                print('fn_ini',fn_ini)
+                fn_mod = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                         str(int(run_station_chunk))+'_end.yaml'
+                file_ini = open(fn_ini,'w')
+                file_mod = open(fn_mod,'w')
+    
+                #iexp = 0
+                onerun = False
+                print('starting station chunk number: '\
+                      +str(run_station_chunk)+'(size: '+str(args.split_by)+' soundings)')
+    
+    
+                isim = 0
+                records_morning_station_chunk = records_morning_station.iloc[((run_station_chunk)*int(args.split_by)):((run_station_chunk+1)*int(args.split_by))] #  [(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+                for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                    #if iexp == 11:
+                    
+                
+                        c4gli_morning = get_record_yaml(file_morning, 
+                                                        record_morning.index_start, 
+                                                        record_morning.index_end,
+                                                        mode='model_input')
+                        if args.diag_tropo is not None:
+                            seltropo = (c4gli_morning.air_ac.p > c4gli_morning.air_ac.p.iloc[-1]+ 3000.*(- 1.2 * 9.81 ))
+                            profile_tropo = c4gli_morning.air_ac[seltropo]
+                            for var in args.diag_tropo:#['t','q','u','v',]:
+                                if var[:3] == 'adv':
+                                    mean_adv_tropo = np.mean(profile_tropo[var+'_x']+profile_tropo[var+'_y'] )
+                                    c4gli_morning.update(source='era-interim',pars={var+'_tropo':mean_adv_tropo})
+                                else:
+                                    print("warning: tropospheric variable "+var+" not recognized")
+                        
+                        #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+                        
+                        if args.runtime == 'from_profile_pair':
+                            record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+                            c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                                              int(record_afternoon.index_start), 
+                                                              int(record_afternoon.index_end),
+                                                            mode='model_input')
+                            runtime = int((c4gli_afternoon.pars.datetime_daylight - 
+                                                 c4gli_morning.pars.datetime_daylight).total_seconds())
+                        elif args.runtime == 'from_input':
+                            runtime = c4gli_morning.pars.runtime
+                        else:
+                            runtime = int(args.runtime)
+    
+                
+                        c4gli_morning.update(source='pairs',pars={'runtime' : \
+                                            runtime})
+
+                        c4gli_morning.update(source=expname, pars=exp)
+    
+                        c4gl = class4gl(c4gli_morning)
+                        
+                        #EFobs = c4gli_morning.pars.BR /(c4gli_morning.pars.BR+1.)
+                        EFobs = c4gli_morning.pars.EF
+                        
+                        b = c4gli_morning.pars.wwilt
+                        c = c4gli_morning.pars.wfc #max(c4gli_morning.pars.wfc,c4gli_morning.pars.wsat-0.01)
+                        
+                        try:
+                            #fb = f(b)
+                            c4gli_morning.pars.wg = b
+                            c4gli_morning.pars.w2 = b
+                            c4gl = class4gl(c4gli_morning)
+                            c4gl.run()
+                            EFmod = c4gl.out.LE.sum()/(c4gl.out.H.sum() + c4gl.out.LE.sum())
+                            fb = EFmod - EFobs
+                            EFmodb = EFmod
+                            c4glb = c4gl
+                            c4gli_morningb = c4gli_morning
+                            
+                            #fc = f(c)
+                            c4gli_morning.pars.wg = c
+                            c4gli_morning.pars.w2 = c
+                            c4gl = class4gl(c4gli_morning)
+                            c4gl.run()
+                            EFmod = c4gl.out.LE.sum()/(c4gl.out.H.sum() + c4gl.out.LE.sum())
+                            fc = EFmod - EFobs
+                            print (EFmodb,EFobs,fb)
+                            print (EFmod,EFobs,fc)
+                            c4glc = c4gl
+                            c4gli_morningc = c4gli_morning
+                            i=0
+                            
+                            if fc*fb > 0.:
+                                if abs(fb) < abs(fc):
+                                    c4gl = c4glb
+                                    c4gli_morning = c4gli_morningb
+                                else:
+                                    c4gl = c4glc
+                                    c4gli_morning = c4gli_morningc
+                                print("Warning!!! function value of the boundaries have the same sign, so I will not able to find a root")
+                            
+                            else:
+                                print('starting ITERATION!!!')
+                                cn  = c - fc/(fc-fb)*(c-b)
+                                
+                                
+                                #fcn = f(cn)
+                                c4gli_morning.pars.wg = np.asscalar(cn)
+                                c4gli_morning.pars.w2 = np.asscalar(cn)
+                                c4gl = class4gl(c4gli_morning)
+                                c4gl.run()
+                                fcn = c4gl.out.LE.sum()/(c4gl.out.H.sum() + c4gl.out.LE.sum()) - EFobs
+                                
+                                tol = 0.02
+                                ftol = 10.
+                                maxiter = 10
+                                
+                                is1=0
+                                is1max=1
+                                while (( abs(cn-c) > tol) or ( abs(fcn) > ftol)) and (fcn != 0) and (i < maxiter):
+                                    if fc * fcn > 0:
+                                        temp = c
+                                        c = b
+                                        b = temp
+                                    
+                                    a = b
+                                    fa = fb
+                                    b = c
+                                    fb = fc
+                                    c = cn
+                                    fc = fcn
+                                                  
+                                    print(i,a,b,c,fcn)
+                                    
+                                    s1 = c - fc/(fc-fb)*(c-b) 
+                                    s2 = c - fc/(fc-fa)*(c-a)
+                                    
+                                    
+                                    # take the one that is closest to the border  (opposite to the previous border), making the chance that the border is eliminated is bigger
+                                    
+                                    
+                                    if (abs(s1-b) < abs(s2-b)):
+                                        is1 = 0
+                                    else:
+                                        is1 +=1
+                                        
+                                    # we prefer s1, but only allow it a few times to not provide the opposite boundary
+                                    if is1 < is1max:           
+                                        s = s1
+                                        print('s1')
+                                    else:
+                                        is1 = 0
+                                        s = s2
+                                        print('s2')
+                                    
+                                    if c > b:
+                                        l = b
+                                        r = c
+                                    else:
+                                        l = c
+                                        r = b
+                                    
+                                    m = (b+c)/2.
+                                         
+                                    if ((s > l) and (s < r)):# and (abs(m-b) < abs(s - b)):
+                                        cn = s
+                                        print('midpoint')
+                                    else:
+                                        cn = m
+                                        print('bissection')
+                                        
+                                    
+                                    #fcn = f(cn)
+                                    c4gli_morning.pars.wg = np.asscalar(cn)
+                                    c4gli_morning.pars.w2 = np.asscalar(cn)
+                                    c4gl = class4gl(c4gli_morning)
+                                    c4gl.run()
+                                    fcn = c4gl.out.LE.sum()/(c4gl.out.H.sum() + c4gl.out.LE.sum()) - EFobs
+                                    
+                                
+                                    i+=1
+                                    
+                                if i == maxiter:
+                                    raise StopIteration('did not converge')
+    
+    
+    
+    
+                            #c4gl = class4gl(c4gli_morning)
+                            #c4gl.run()
+    
+                            c4gli_morning.pars.itersteps = i
+                            c4gli_morning.dump(file_ini)
+                            
+                            
+                            c4gl.dump(file_mod,\
+                                          include_input=False,\
+                                       #   timeseries_only=timeseries_only,\
+                                     )
+                            onerun = True
+                        except:
+                            print('run not succesfull')
+    
+                    #iexp = iexp +1
+                file_ini.close()
+                file_mod.close()
+                file_morning.close()
+                if args.runtime == 'from_profile_pair':
+                    file_afternoon.close()
+        
+                if onerun:
+                    records_ini = get_records(pd.DataFrame([current_station]),\
+                                                               path_exp,\
+                                                               getchunk = int(run_station_chunk),\
+                                                               subset='ini',
+                                                               refetch_records=True,
+                                                               )
+                    records_mod = get_records(pd.DataFrame([current_station]),\
+                                                               path_exp,\
+                                                               getchunk = int(run_station_chunk),\
+                                                               subset='end',\
+                                                               refetch_records=True,\
+                                                               )
+                else:
+                    # remove empty files
+                    os.system('rm '+fn_ini)
+                    os.system('rm '+fn_mod)
+        
+        # # align afternoon records with initial records, and set same index
+        # records_afternoon.index = records_afternoon.ldatetime.dt.date
+        # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+        # records_afternoon.index = records_ini.index
+        
+        # stations_for_iter = stations(path_exp)
+        # for STNID,station in stations_iterator(stations_for_iter):
+        #     records_current_station_index = \
+        #             (records_ini.index.get_level_values('STNID') == STNID)
+        #     file_current_station_mod = STNID
+        # 
+        #     with \
+        #     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+        #     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+        #     open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+        #         for (STNID,index),record_ini in records_iterator(records_ini):
+        #             c4gli_ini = get_record_yaml(file_station_ini, 
+        #                                         record_ini.index_start, 
+        #                                         record_ini.index_end,
+        #                                         mode='ini')
+        #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+        # 
+        #             record_mod = records_mod.loc[(STNID,index)]
+        #             c4gl_mod = get_record_yaml(file_station_mod, 
+        #                                         record_mod.index_start, 
+        #                                         record_mod.index_end,
+        #                                         mode='mod')
+        #             record_afternoon = records_afternoon.loc[(STNID,index)]
+        #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+        #                                         record_afternoon.index_start, 
+        #                                         record_afternoon.index_end,
+        #                                         mode='ini')
+    
+if __name__ == '__main__':
+    #execute(**vars(args))
+    execute()
diff --git a/class4gl/simulations/simulations_iter_bowen.py b/class4gl/simulations/simulations_iter_bowen.py
new file mode 100644
index 0000000..9417534
--- /dev/null
+++ b/class4gl/simulations/simulations_iter_bowen.py
@@ -0,0 +1,475 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--error_handling',default='dump_on_success')
+parser.add_argument('--diag_tropo',default=['advt','advq','advu','advv'])
+parser.add_argument('--subset_forcing',default='morning') # this tells which yaml subset
+                                                      # to initialize with.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime',default='from_afternoon_profile')
+
+parser.add_argument('--experiments')
+parser.add_argument('--split_by',default=-1)# station soundings are split
+                                            # up in chunks
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+EXP_DEFS  =\
+{
+  'ERA_NOAC_ITER':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'NOAC_ITER':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ADV_ITER':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'W_ITER':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'AC_ITER': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC_ITER':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_ITER_BOWEN':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_W_ITER':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_AC_ITER': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+  'IOPS_NOAC_ITER':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'IOPS_ADV_ITER':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'IOPS_W_ITER':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'IOPS_AC_ITER': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+
+# #SET = 'GLOBAL'
+# SET = args.dataset
+
+# ========================
+print("getting a list of stations")
+# ========================
+
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+
+# ====================================
+print('defining all_stations_select')
+# ====================================
+
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+
+
+
+if args.station_id is not None:
+    print("Selecting station by ID")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print(all_stations_select)
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         args.path_forcing,\
+                                         subset=args.subset_forcing,
+                                         refetch_records=False,
+                                         )
+
+# only run a specific chunck from the selection
+if args.global_chunk_number is not None:
+    if args.station_chunk_number is not None:
+        raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+
+
+    if not (int(args.split_by) > 0) :
+            raise ValueError("global_chunk_number is specified, but --split-by is not a strict positive number, so I don't know how to split the batch into chunks.")
+
+    run_station_chunk = None
+    print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+    totalchunks = 0
+    stations_iter = all_stations_select.iterrows()
+    in_current_chunk = False
+    try:
+        while not in_current_chunk:
+            istation,current_station = stations_iter.__next__()
+            all_records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+            chunks_current_station = math.ceil(float(len(all_records_morning_station_select))/float(args.split_by))
+            print('chunks_current_station',chunks_current_station)
+            in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+        
+            if in_current_chunk:
+                run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                run_station_chunk = int(args.global_chunk_number) - totalchunks 
+        
+            totalchunks +=chunks_current_station
+        
+
+    except StopIteration:
+       raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+    print("station = ",list(run_stations.index))
+    print("station chunk number:",run_station_chunk)
+
+# if no global chunk is specified, then run the whole station selection in one run, or
+# a specific chunk for each selected station according to # args.station_chunk_number
+else:
+    run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+    if args.station_chunk_number is not None:
+        run_station_chunk = int(args.station_chunk_number)
+        print("station(s) that is processed.",list(run_stations.index))
+        print("chunk number: ",run_station_chunk)
+    else:
+        if args.split_by != -1:
+            raise ValueError("Chunks are defined by --split-by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split-by.")
+        run_station_chunk = 0
+        print("stations that are processed.",list(run_stations.index))
+        
+
+#print(all_stations)
+print('Fetching initial/forcing records')
+records_morning = get_records(run_stations,\
+                              args.path_forcing,\
+                              subset=args.subset_forcing,
+                              refetch_records=False,
+                              )
+
+# note that if runtime is an integer number, we don't need to get the afternoon
+# profiles. 
+if args.runtime == 'from_afternoon_profile':
+    print('Fetching afternoon records for determining the simulation runtimes')
+    records_afternoon = get_records(run_stations,\
+                                    args.path_forcing,\
+                                    subset='afternoon',
+                                    refetch_records=False,
+                                    )
+    
+    # print(records_morning.index)
+    # print(records_afternoon.index)
+    # align afternoon records with the noon records, and set same index
+    print('hello')
+    print(len(records_afternoon))
+    print(len(records_morning))
+
+    print("aligning morning and afternoon records")
+    records_morning['dates'] = records_morning['ldatetime'].dt.date
+    records_afternoon['dates'] = records_afternoon['ldatetime'].dt.date
+    records_afternoon.set_index(['STNID','dates'],inplace=True)
+    ini_index_dates = records_morning.set_index(['STNID','dates']).index
+    records_afternoon = records_afternoon.loc[ini_index_dates]
+    records_afternoon.index = records_morning.index
+
+experiments = args.experiments.strip(' ').split(' ')
+for expname in experiments:
+    exp = EXP_DEFS[expname]
+    path_exp = args.path_experiments+'/'+expname+'/'
+
+    os.system('mkdir -p '+path_exp)
+    records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+    for istation,current_station in run_stations.iterrows():
+        if (int(args.split_by) * int(run_station_chunk)) >= (len(records_morning_station)):
+            print("warning: outside of profile number range for station "+\
+                  str(current_station)+". Skipping chunk number for this station.")
+        else:
+
+            fn_morning = args.path_forcing+'/'+format(current_station.name,'05d')+'_'+args.subset_forcing+'.yaml'
+            if os.path.isfile(fn_morning):
+                file_morning = open(fn_morning)
+            else:
+                fn_morning = \
+                     args.path_forcing+'/'+format(current_station.name,'05d')+\
+                     '_'+str(run_station_chunk)+'_'+args.subset_forcing+'.yaml'
+                file_morning = open(fn_morning)
+
+            if args.runtime == 'from_afternoon_profile':
+                file_afternoon = open(args.path_forcing+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+            fn_ini = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_ini.yaml'
+            fn_mod = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_mod.yaml'
+            file_ini = open(fn_ini,'w')
+            file_mod = open(fn_mod,'w')
+
+            #iexp = 0
+            onerun = False
+            print('starting station chunk number: '\
+                  +str(run_station_chunk)+'(size: '+str(args.split_by)+' soundings)')
+
+
+            isim = 0
+            records_morning_station_chunk = records_morning_station.iloc[((run_station_chunk)*int(args.split_by)):((run_station_chunk+1)*int(args.split_by))] #  [(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+            for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                #if iexp == 11:
+                
+            
+                    c4gli_morning = get_record_yaml(file_morning, 
+                                                    record_morning.index_start, 
+                                                    record_morning.index_end,
+                                                    mode='ini')
+                    if args.diag_tropo is not None:
+                        seltropo = (c4gli_morning.air_ac.p > c4gli_morning.air_ac.p.iloc[-1]+ 3000.*(- 1.2 * 9.81 ))
+                        profile_tropo = c4gli_morning.air_ac[seltropo]
+                        for var in args.diag_tropo:#['t','q','u','v',]:
+                            if var[:3] == 'adv':
+                                mean_adv_tropo = np.mean(profile_tropo[var+'_x']+profile_tropo[var+'_y'] )
+                                c4gli_morning.update(source='era-interim',pars={var+'_tropo':mean_adv_tropo})
+                            else:
+                                print("warning: tropospheric variable "+var+" not recognized")
+                    
+                    #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+                    
+                    
+                    record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+                    c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                                      record_afternoon.index_start, 
+                                                      record_afternoon.index_end,
+                                                    mode='ini')
+            
+                    c4gli_morning.update(source='pairs',pars={'runtime' : \
+                                        int((c4gli_afternoon.pars.datetime_daylight - 
+                                             c4gli_morning.pars.datetime_daylight).total_seconds())})
+                    c4gli_morning.update(source=expname, pars=exp)
+
+                    c4gl = class4gl(c4gli_morning)
+                    
+                    #EFobs = c4gli_morning.pars.BR /(c4gli_morning.pars.BR+1.)
+                    EFobs = (1.-c4gli_morning.pars.EF)/c4gli_morning.pars.EF
+                    
+                    b = c4gli_morning.pars.wwilt
+                    c = c4gli_morning.pars.wfc #max(c4gli_morning.pars.wfc,c4gli_morning.pars.wsat-0.01)
+                    
+                    
+                    try:
+                        #fb = f(b)
+                        c4gli_morning.pars.wg = b
+                        c4gli_morning.pars.w2 = b
+                        c4gl = class4gl(c4gli_morning)
+                        c4gl.run()
+                        EFmod = c4gl.out.H.sum()/(c4gl.out.LE.sum())
+                        fb = EFmod - EFobs
+                        EFmodb = EFmod
+                        c4glb = c4gl
+                        c4gli_morningb = c4gli_morning
+                        
+                        #fc = f(c)
+                        c4gli_morning.pars.wg = c
+                        c4gli_morning.pars.w2 = c
+                        c4gl = class4gl(c4gli_morning)
+                        c4gl.run()
+                        EFmod = c4gl.out.H.sum()/(c4gl.out.LE.sum())
+                        fc = EFmod - EFobs
+                        print (EFmodb,EFobs,fb)
+                        print (EFmod,EFobs,fc)
+                        c4glc = c4gl
+                        c4gli_morningc = c4gli_morning
+                        i=0
+                        
+
+                        if fc*fb > 0.:
+                            if abs(fb) < abs(fc):
+                                c4gl = c4glb
+                                c4gli_morning = c4gli_morningb
+                            else:
+                                c4gl = c4glc
+                                c4gli_morning = c4gli_morningc
+                            print("Warning!!! function value of the boundaries have the same sign, so I will not able to find a root")
+                        
+                        else:
+                            print('starting ITERATION!!!')
+                            cn  = c - fc/(fc-fb)*(c-b)
+                            
+                            
+                            #fcn = f(cn)
+                            c4gli_morning.pars.wg = np.asscalar(cn)
+                            c4gli_morning.pars.w2 = np.asscalar(cn)
+                            c4gl = class4gl(c4gli_morning)
+                            c4gl.run()
+                            fcn = c4gl.out.H.sum()/c4gl.out.LE.sum() - EFobs
+                            
+                            tol = 0.02
+                            ftol = 10.
+                            maxiter = 10
+                            
+                            is1=0
+                            is1max=1
+                            while (( abs(cn-c) > tol) or ( abs(fcn) > ftol)) and (fcn != 0) and (i < maxiter):
+                                if fc * fcn > 0:
+                                    temp = c
+                                    c = b
+                                    b = temp
+                                
+                                a = b
+                                fa = fb
+                                b = c
+                                fb = fc
+                                c = cn
+                                fc = fcn
+                                              
+                                print(i,a,b,c,fcn)
+                                
+                                s1 = c - fc/(fc-fb)*(c-b) 
+                                s2 = c - fc/(fc-fa)*(c-a)
+                                
+                                
+                                # take the one that is closest to the border  (opposite to the previous border), making the chance that the border is eliminated is bigger
+                                
+                                
+                                if (abs(s1-b) < abs(s2-b)):
+                                    is1 = 0
+                                else:
+                                    is1 +=1
+                                    
+                                # we prefer s1, but only allow it a few times to not provide the opposite boundary
+                                if is1 < is1max:           
+                                    s = s1
+                                    print('s1')
+                                else:
+                                    is1 = 0
+                                    s = s2
+                                    print('s2')
+                                
+                                if c > b:
+                                    l = b
+                                    r = c
+                                else:
+                                    l = c
+                                    r = b
+                                
+                                m = (b+c)/2.
+                                     
+                                if ((s > l) and (s < r)):# and (abs(m-b) < abs(s - b)):
+                                    cn = s
+                                    print('midpoint')
+                                else:
+                                    cn = m
+                                    print('bissection')
+                                    
+                                
+                                #fcn = f(cn)
+                                c4gli_morning.pars.wg = np.asscalar(cn)
+                                c4gli_morning.pars.w2 = np.asscalar(cn)
+                                c4gl = class4gl(c4gli_morning)
+                                c4gl.run()
+                                fcn = c4gl.out.H.sum()/c4gl.out.LE.sum() - EFobs
+                                
+                            
+                                i+=1
+                                
+                            if i == maxiter:
+                                raise StopIteration('did not converge')
+
+
+
+
+                        #c4gl = class4gl(c4gli_morning)
+                        #c4gl.run()
+
+                        c4gli_morning.pars.itersteps = i
+                        c4gli_morning.dump(file_ini)
+                        
+                        
+                        c4gl.dump(file_mod,\
+                                      include_input=False,\
+                                   #   timeseries_only=timeseries_only,\
+                                 )
+                        onerun = True
+                    except:
+                        print('run not succesfull')
+
+                #iexp = iexp +1
+            file_ini.close()
+            file_mod.close()
+            file_morning.close()
+            file_afternoon.close()
+    
+            if onerun:
+                records_ini = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='ini',
+                                                           refetch_records=True,
+                                                           )
+                records_mod = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='mod',\
+                                                           refetch_records=True,\
+                                                           )
+            else:
+                # remove empty files
+                os.system('rm '+fn_ini)
+                os.system('rm '+fn_mod)
+    
+    # # align afternoon records with initial records, and set same index
+    # records_afternoon.index = records_afternoon.ldatetime.dt.date
+    # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+    # records_afternoon.index = records_ini.index
+    
+    # stations_for_iter = stations(path_exp)
+    # for STNID,station in stations_iterator(stations_for_iter):
+    #     records_current_station_index = \
+    #             (records_ini.index.get_level_values('STNID') == STNID)
+    #     file_current_station_mod = STNID
+    # 
+    #     with \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+    #     open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+    #         for (STNID,index),record_ini in records_iterator(records_ini):
+    #             c4gli_ini = get_record_yaml(file_station_ini, 
+    #                                         record_ini.index_start, 
+    #                                         record_ini.index_end,
+    #                                         mode='ini')
+    #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+    # 
+    #             record_mod = records_mod.loc[(STNID,index)]
+    #             c4gl_mod = get_record_yaml(file_station_mod, 
+    #                                         record_mod.index_start, 
+    #                                         record_mod.index_end,
+    #                                         mode='mod')
+    #             record_afternoon = records_afternoon.loc[(STNID,index)]
+    #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+    #                                         record_afternoon.index_start, 
+    #                                         record_afternoon.index_end,
+    #                                         mode='ini')
+
diff --git a/class4gl/simulations/simulations_iter_test.py b/class4gl/simulations/simulations_iter_test.py
new file mode 100644
index 0000000..eefd475
--- /dev/null
+++ b/class4gl/simulations/simulations_iter_test.py
@@ -0,0 +1,367 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+sys.path.insert(0, '/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/')
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+
+EXP_DEFS  =\
+{
+  'ITER_NOAC':{'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ITER_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'ITER_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'ITER_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+import argparse
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--global-chunk')
+    parser.add_argument('--first-station')
+    parser.add_argument('--last-station')
+    parser.add_argument('--dataset')
+    parser.add_argument('--path-soundings')
+    parser.add_argument('--experiments')
+    parser.add_argument('--split-by',default=-1)# station soundings are split
+                                                # up in chunks
+    parser.add_argument('--station-chunk')
+    args = parser.parse_args()
+
+
+#SET = 'GLOBAL'
+SET = args.dataset
+
+if 'path-soundings' in args.__dict__.keys():
+    path_soundingsSET = args.__dict__['path-soundings']+'/'+SET+'/'
+else:
+    path_soundingsSET = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/'+SET+'/'
+
+all_stations = stations(path_soundingsSET,suffix='morning',refetch_stations=True).table
+
+all_records_morning = get_records(all_stations,\
+                              path_soundingsSET,\
+                              subset='morning',
+                              refetch_records=False,
+                              )
+
+if args.global_chunk is not None:
+    totalchunks = 0
+    stations_iterator = all_stations.iterrows()
+    in_current_chunk = False
+    while not in_current_chunk:
+        istation,current_station = stations_iterator.__next__()
+        all_records_morning_station = all_records_morning.query('STNID == '+str(current_station.name))
+        chunks_current_station = math.ceil(float(len(all_records_morning_station))/float(args.split_by))
+        in_current_chunk = (int(args.global_chunk) < (totalchunks+chunks_current_station))
+
+        if in_current_chunk:
+            run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+            run_station_chunk = int(args.global_chunk) - totalchunks 
+
+        totalchunks +=chunks_current_station
+
+else:
+    run_stations = pd.DataFrame(all_stations)
+    if args.last_station is not None:
+        run_stations = run_stations.iloc[:(int(args.__dict__['last_station'])+1)]
+    if args.first_station is not None:
+        run_stations = run_stations.iloc[int(args.__dict__['first_station']):]
+    run_station_chunk = 0
+    if args.station_chunk is not None:
+        run_station_chunk = args.station_chunk
+
+#print(all_stations)
+print(run_stations)
+print(args.__dict__.keys())
+records_morning = get_records(run_stations,\
+                              path_soundingsSET,\
+                              subset='morning',
+                              refetch_records=False,
+                              )
+records_afternoon = get_records(run_stations,\
+                                path_soundingsSET,\
+                                subset='afternoon',
+                                refetch_records=False,
+                                )
+
+# align afternoon records with the noon records, and set same index
+records_afternoon.index = records_afternoon.ldatetime.dt.date
+records_afternoon = records_afternoon.loc[records_morning.ldatetime.dt.date]
+records_afternoon.index = records_morning.index
+
+experiments = args.experiments.split(';')
+
+for expname in experiments:
+    exp = EXP_DEFS[expname]
+    path_exp = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/'+SET+'_'+expname+'/'
+
+    os.system('mkdir -p '+path_exp)
+    for istation,current_station in run_stations.iterrows():
+        records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+        if (int(args.split_by) * int(run_station_chunk)) >= (len(records_morning_station)):
+            print("warning: outside of profile number range for station "+\
+                  str(current_station)+". Skipping chunk number for this station.")
+        else:
+            file_morning = open(path_soundingsSET+'/'+format(current_station.name,'05d')+'_morning.yaml')
+            file_afternoon = open(path_soundingsSET+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+            fn_ini = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_ini.yaml'
+            fn_mod = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_mod.yaml'
+            file_ini = open(fn_ini,'w')
+            file_mod = open(fn_mod,'w')
+
+            #iexp = 0
+            onerun = False
+
+            records_morning_station_chunk = records_morning_station[(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+                #if iexp == 11:
+            for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                
+            
+                    c4gli_morning = get_record_yaml(file_morning, 
+                                                    record_morning.index_start, 
+                                                    record_morning.index_end,
+                                                    mode='ini')
+                    
+                    #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+                    
+                    
+                    record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+                    c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                                      record_afternoon.index_start, 
+                                                      record_afternoon.index_end,
+                                                    mode='ini')
+            
+                    c4gli_morning.update(source='pairs',pars={'runtime' : \
+                                        int((c4gli_afternoon.pars.datetime_daylight - 
+                                             c4gli_morning.pars.datetime_daylight).total_seconds())})
+                    c4gli_morning.update(source=expname, pars=exp)
+
+                    c4gl = class4gl(c4gli_morning)
+                    
+                    #EFobs = c4gli_morning.pars.BR /(c4gli_morning.pars.BR+1.)
+                    EFobs = c4gli_morning.pars.EF
+                    
+                    b = c4gli_morning.pars.wwilt
+                    c = c4gli_morning.pars.wfc #max(c4gli_morning.pars.wfc,c4gli_morning.pars.wsat-0.01)
+                    
+                    
+                    try:
+                        #fb = f(b)
+                        c4gli_morning.pars.wg = b
+                        c4gli_morning.pars.w2 = b
+                        c4gl = class4gl(c4gli_morning)
+                        c4gl.run()
+                        EFmod = c4gl.out.LE.sum()/(c4gl.out.H.sum() + c4gl.out.LE.sum())
+                        fb = EFmod - EFobs
+                        EFmodb = EFmod
+                        c4glb = c4gl
+                        c4gli_morningb = c4gli_morning
+                        
+                        #fc = f(c)
+                        c4gli_morning.pars.wg = c
+                        c4gli_morning.pars.w2 = c
+                        c4gl = class4gl(c4gli_morning)
+                        c4gl.run()
+                        EFmod = c4gl.out.LE.sum()/(c4gl.out.H.sum() + c4gl.out.LE.sum())
+                        fc = EFmod - EFobs
+                        print (EFmodb,EFobs,fb)
+                        print (EFmod,EFobs,fc)
+                        c4glc = c4gl
+                        c4gli_morningc = c4gli_morning
+                        i=0
+                        
+
+                        if fc*fb > 0.:
+                            if abs(fb) < abs(fc):
+                                c4gl = c4glb
+                                c4gli_morning = c4gli_morningb
+                            else:
+                                c4gl = c4glc
+                                c4gli_morning = c4gli_morningc
+                            print("Warning!!! function value of the boundaries have the same sign, so I will not able to find a root")
+                        
+                        else:
+                            print('starting ITERATION!!!')
+                            cn  = c - fc/(fc-fb)*(c-b)
+                            
+                            
+                            #fcn = f(cn)
+                            c4gli_morning.pars.wg = np.asscalar(cn)
+                            c4gli_morning.pars.w2 = np.asscalar(cn)
+                            c4gl = class4gl(c4gli_morning)
+                            c4gl.run()
+                            fcn = c4gl.out.LE.sum()/(c4gl.out.H.sum() + c4gl.out.LE.sum()) - EFobs
+                            
+                            tol = 0.02
+                            ftol = 10.
+                            maxiter = 10
+                            
+                            is1=0
+                            is1max=1
+                            while (( abs(cn-c) > tol) or ( abs(fcn) > ftol)) and (fcn != 0) and (i < maxiter):
+                                if fc * fcn > 0:
+                                    temp = c
+                                    c = b
+                                    b = temp
+                                
+                                a = b
+                                fa = fb
+                                b = c
+                                fb = fc
+                                c = cn
+                                fc = fcn
+                                              
+                                print(i,a,b,c,fcn)
+                                
+                                s1 = c - fc/(fc-fb)*(c-b) 
+                                s2 = c - fc/(fc-fa)*(c-a)
+                                
+                                
+                                # take the one that is closest to the border  (opposite to the previous border), making the chance that the border is eliminated is bigger
+                                
+                                
+                                if (abs(s1-b) < abs(s2-b)):
+                                    is1 = 0
+                                else:
+                                    is1 +=1
+                                    
+                                # we prefer s1, but only allow it a few times to not provide the opposite boundary
+                                if is1 < is1max:           
+                                    s = s1
+                                    print('s1')
+                                else:
+                                    is1 = 0
+                                    s = s2
+                                    print('s2')
+                                
+                                if c > b:
+                                    l = b
+                                    r = c
+                                else:
+                                    l = c
+                                    r = b
+                                
+                                m = (b+c)/2.
+                                     
+                                if ((s > l) and (s < r)):# and (abs(m-b) < abs(s - b)):
+                                    cn = s
+                                    print('midpoint')
+                                else:
+                                    cn = m
+                                    print('bissection')
+                                    
+                                
+                                #fcn = f(cn)
+                                c4gli_morning.pars.wg = np.asscalar(cn)
+                                c4gli_morning.pars.w2 = np.asscalar(cn)
+                                c4gl = class4gl(c4gli_morning)
+                                c4gl.run()
+                                fcn = c4gl.out.LE.sum()/(c4gl.out.H.sum() + c4gl.out.LE.sum()) - EFobs
+                                
+                            
+                                i+=1
+                                
+                            if i == maxiter:
+                                raise StopIteration('did not converge')
+
+
+
+
+                        #c4gl = class4gl(c4gli_morning)
+                        #c4gl.run()
+                        onerun = True
+
+                        c4gli_morning.pars.itersteps = i
+                    except:
+                        print('run not succesfull')
+                    c4gli_morning.dump(file_ini)
+                    
+                    
+                    c4gl.dump(file_mod,\
+                                  include_input=False,\
+                               #   timeseries_only=timeseries_only,\
+                             )
+                    onerun = True
+
+                #iexp = iexp +1
+            file_ini.close()
+            file_mod.close()
+            file_morning.close()
+            file_afternoon.close()
+    
+            if onerun:
+                records_ini = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='ini',
+                                                           refetch_records=True,
+                                                           )
+                records_mod = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='mod',\
+                                                           refetch_records=True,\
+                                                           )
+            else:
+                # remove empty files
+                os.system('rm '+fn_ini)
+                os.system('rm '+fn_mod)
+    
+    # # align afternoon records with initial records, and set same index
+    # records_afternoon.index = records_afternoon.ldatetime.dt.date
+    # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+    # records_afternoon.index = records_ini.index
+    
+    # stations_for_iter = stations(path_exp)
+    # for STNID,station in stations_iterator(stations_for_iter):
+    #     records_current_station_index = \
+    #             (records_ini.index.get_level_values('STNID') == STNID)
+    #     file_current_station_mod = STNID
+    # 
+    #     with \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+    #     open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+    #         for (STNID,index),record_ini in records_iterator(records_ini):
+    #             c4gli_ini = get_record_yaml(file_station_ini, 
+    #                                         record_ini.index_start, 
+    #                                         record_ini.index_end,
+    #                                         mode='ini')
+    #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+    # 
+    #             record_mod = records_mod.loc[(STNID,index)]
+    #             c4gl_mod = get_record_yaml(file_station_mod, 
+    #                                         record_mod.index_start, 
+    #                                         record_mod.index_end,
+    #                                         mode='mod')
+    #             record_afternoon = records_afternoon.loc[(STNID,index)]
+    #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+    #                                         record_afternoon.index_start, 
+    #                                         record_afternoon.index_end,
+    #                                         mode='ini')
+
diff --git a/class4gl/simulations/simulations_smchange2.py b/class4gl/simulations/simulations_smchange2.py
new file mode 100644
index 0000000..d79a46f
--- /dev/null
+++ b/class4gl/simulations/simulations_smchange2.py
@@ -0,0 +1,358 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--error_handling',default='dump_on_success')
+parser.add_argument('--subset_forcing',default='morning') # this tells which yaml subset
+                                                      # to initialize with.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime',default='from_afternoon_profile')
+
+parser.add_argument('--experiments')
+parser.add_argument('--split_by',default=-1)# station soundings are split
+                                            # up in chunks
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+EXP_DEFS  =\
+{
+  'SMa':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.05},
+  'SMb':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.1},
+  'SMc':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.15},
+  'SMd':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.02},
+  'SMe':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.25},
+  'SMf':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.3},
+  'SMg':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.35},
+  'SMh':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.4},
+  'SMi':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.45},
+  'SMj':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.5},
+  'SMk':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.55},
+  'SMl':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.6},
+  'SMm':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.65},
+  'SMo':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.7},
+  'SMp':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.75},
+  'SMq':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.08},
+  'SMr':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.85},
+  'SMs':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.9},
+  'SMt':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 0.95},
+  'SMu':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange': 1.},
+}
+
+##for i in range(1,20):
+##    EXP_DEFS['SM'+str(i)] = {'sw_ac' : [],'sw_ap': True,'sw_lit': False, 'wgchange':float(i)/20.}
+
+
+
+# #SET = 'GLOBAL'
+# SET = args.dataset
+
+
+print("getting stations")
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+
+print('defining all_stations_select')
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+if args.station_id is not None:
+    print("Selecting station by ID")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         args.path_forcing,\
+                                         subset=args.subset_forcing,
+                                         refetch_records=False,
+                                         )
+
+# only run a specific chunck from the selection
+if args.global_chunk_number is not None:
+    if args.station_chunk_number is not None:
+        raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+
+
+    if not (int(args.split_by) > 0) :
+            raise ValueError("global_chunk_number is specified, but --split-by is not a strict positive number, so I don't know how to split the batch into chunks.")
+
+    run_station_chunk = None
+    print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+    totalchunks = 0
+    stations_iter = all_stations_select.iterrows()
+    in_current_chunk = False
+    try:
+        while not in_current_chunk:
+            istation,current_station = stations_iter.__next__()
+            all_records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+            chunks_current_station = math.ceil(float(len(all_records_morning_station_select))/float(args.split_by))
+            print('chunks_current_station',chunks_current_station)
+            in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+        
+            if in_current_chunk:
+                run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                run_station_chunk = int(args.global_chunk_number) - totalchunks 
+        
+            totalchunks +=chunks_current_station
+        
+
+    except StopIteration:
+       raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+    print("station = ",list(run_stations.index))
+    print("station chunk number:",run_station_chunk)
+
+# if no global chunk is specified, then run the whole station selection in one run, or
+# a specific chunk for each selected station according to # args.station_chunk_number
+else:
+    run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+    if args.station_chunk_number is not None:
+        run_station_chunk = int(args.station_chunk_number)
+        print("station(s) that is processed.",list(run_stations.index))
+        print("chunk number: ",run_station_chunk)
+    else:
+        if args.split_by != -1:
+            raise ValueError("Chunks are defined by --split-by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split-by.")
+        run_station_chunk = 0
+        print("stations that are processed.",list(run_stations.index))
+        
+
+#print(all_stations)
+print('Fetching initial/forcing records')
+records_morning = get_records(run_stations,\
+                              args.path_forcing,\
+                              subset=args.subset_forcing,
+                              refetch_records=False,
+                              )
+
+# note that if runtime is an integer number, we don't need to get the afternoon
+# profiles. 
+if args.runtime == 'from_afternoon_profile':
+    print('Fetching afternoon records for determining the simulation runtimes')
+    records_afternoon = get_records(run_stations,\
+                                    args.path_forcing,\
+                                    subset='afternoon',
+                                    refetch_records=False,
+                                    )
+    
+    # print(records_morning.index)
+    # print(records_afternoon.index)
+    # align afternoon records with the noon records, and set same index
+    print('hello')
+    print(len(records_afternoon))
+    print(len(records_morning))
+
+    print("aligning morning and afternoon records")
+    records_morning['dates'] = records_morning.ldatetime.dt.date
+    records_afternoon['dates'] = records_afternoon.ldatetime.dt.date
+    records_afternoon.set_index(['STNID','dates'],inplace=True)
+    ini_index_dates = records_morning.set_index(['STNID','dates']).index
+    records_afternoon = records_afternoon.loc[ini_index_dates]
+    records_afternoon.index = records_morning.index
+
+experiments = args.experiments.strip(' ').split(' ')
+for expname in experiments:
+    exp = EXP_DEFS[expname]
+    path_exp = args.path_experiments+'/'+expname+'/'
+
+    os.system('mkdir -p '+path_exp)
+    for istation,current_station in run_stations.iterrows():
+        records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+        if (int(args.split_by) * int(run_station_chunk)) >= (len(records_morning_station)):
+            print("warning: outside of profile number range for station "+\
+                  str(current_station)+". Skipping chunk number for this station.")
+        else:
+            file_morning = open(args.path_forcing+'/'+format(current_station.name,'05d')+'_morning.yaml')
+            file_afternoon = open(args.path_forcing+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+            fn_ini = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_ini.yaml'
+            fn_mod = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_mod.yaml'
+            file_ini = open(fn_ini,'w')
+            file_mod = open(fn_mod,'w')
+
+            #iexp = 0
+            onerun = False
+            print('starting station chunk number: '\
+                  +str(run_station_chunk)+'(size: '+str(args.split_by)+' soundings)')
+
+            records_morning_station_chunk = records_morning_station[(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+
+            isim = 0
+            for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                    print('starting '+str(isim+1)+' out of '+\
+                      str(len(records_morning_station_chunk) )+\
+                      ' (station total: ',str(len(records_morning_station)),')')  
+                
+            
+                    c4gli_morning = get_record_yaml(file_morning, 
+                                                    record_morning.index_start, 
+                                                    record_morning.index_end,
+                                                    mode='ini')
+                    
+                    #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+
+                    sm = exp['wgchange']*(c4gli_morning.pars.wfc - c4gli_morning.pars.wwilt) + c4gli_morning.pars.wwilt
+                    c4gli_morning.update(source=expname, pars={'wg': sm})
+                    c4gli_morning.update(source=expname, pars={'w2': sm})
+                    
+                    
+                    if args.runtime == 'from_afternoon_profile':
+                        record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+                        c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                                          record_afternoon.index_start, 
+                                                          record_afternoon.index_end,
+                                                        mode='ini')
+                        runtime = int((c4gli_afternoon.pars.datetime_daylight - 
+                                             c4gli_morning.pars.datetime_daylight).total_seconds())
+                    else:
+                        runtime = int(args.runtime)
+
+            
+                    c4gli_morning.update(source='pairs',pars={'runtime' : \
+                                        runtime})
+                    c4gli_morning.update(source=expname, pars=exp)
+
+                    c4gl = class4gl(c4gli_morning)
+
+                    if args.error_handling == 'dump_always':
+                        try:
+                            c4gl.run()
+                            print('run succesfull')
+                        except:
+                            print('run not succesfull')
+                        onerun = True
+
+                        c4gli_morning.dump(file_ini)
+                        
+                        
+                        c4gl.dump(file_mod,\
+                                  include_input=False,\
+                                  #timeseries_only=timeseries_only,\
+                                 )
+                        onerun = True
+                    # in this case, only the file will dumped if the runs were
+                    # successful
+                    elif args.error_handling == 'dump_on_success':
+                        try:
+                            c4gl.run()
+                            print('run succesfull')
+                            c4gli_morning.dump(file_ini)
+                            
+                            
+                            c4gl.dump(file_mod,\
+                                      include_input=False,\
+                                      #timeseries_only=timeseries_only,\
+                                     )
+                            onerun = True
+                        except:
+                            print('run not succesfull')
+                    isim += 1
+
+
+            file_ini.close()
+            file_mod.close()
+            file_morning.close()
+            file_afternoon.close()
+    
+            if onerun:
+                records_ini = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='ini',
+                                                           refetch_records=True,
+                                                           )
+                records_mod = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='mod',\
+                                                           refetch_records=True,\
+                                                           )
+            else:
+                # remove empty files
+                os.system('rm '+fn_ini)
+                os.system('rm '+fn_mod)
+    
+    # # align afternoon records with initial records, and set same index
+    # records_afternoon.index = records_afternoon.ldatetime.dt.date
+    # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+    # records_afternoon.index = records_ini.index
+    
+    # stations_for_iter = stations(path_exp)
+    # for STNID,station in stations_iterator(stations_for_iter):
+    #     records_current_station_index = \
+    #             (records_ini.index.get_level_values('STNID') == STNID)
+    #     file_current_station_mod = STNID
+    # 
+    #     with \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+    #     open(path_forcing+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+    #         for (STNID,index),record_ini in records_iterator(records_ini):
+    #             c4gli_ini = get_record_yaml(file_station_ini, 
+    #                                         record_ini.index_start, 
+    #                                         record_ini.index_end,
+    #                                         mode='ini')
+    #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+    # 
+    #             record_mod = records_mod.loc[(STNID,index)]
+    #             c4gl_mod = get_record_yaml(file_station_mod, 
+    #                                         record_mod.index_start, 
+    #                                         record_mod.index_end,
+    #                                         mode='mod')
+    #             record_afternoon = records_afternoon.loc[(STNID,index)]
+    #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+    #                                         record_afternoon.index_start, 
+    #                                         record_afternoon.index_end,
+    #                                         mode='ini')
+
diff --git a/class4gl/simulations/simulations_veg.py b/class4gl/simulations/simulations_veg.py
new file mode 100644
index 0000000..6dbcdac
--- /dev/null
+++ b/class4gl/simulations/simulations_veg.py
@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--error_handling',default='dump_on_success')
+parser.add_argument('--diag_tropo',default=['advt','advq','advu','advv'])
+parser.add_argument('--subset_forcing',default='morning') # this tells which yaml subset
+                                                      # to initialize with.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime',default='from_afternoon_profile')
+
+parser.add_argument('--experiments')
+parser.add_argument('--split_by',default=-1)# station soundings are split
+                                            # up in chunks
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+EXP_DEFS  =\
+{
+  'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC_WILT':   {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC_FC':     {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV':         {'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_VMIN':    {'sw_ac' : ['adv'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_VMAX':    {'sw_ac' : ['adv'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_V0':    {'sw_ac' : ['adv'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_L001':    {'sw_ac' : ['adv'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_L100':    {'sw_ac' : ['adv'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_L099':    {'sw_ac' : ['adv'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+# ========================
+print("getting a list of stations")
+# ========================
+
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+
+# ====================================
+print('defining all_stations_select')
+# ====================================
+
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+
+
+
+if args.station_id is not None:
+    print("Selecting station by ID")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print(all_stations_select)
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         args.path_forcing,\
+                                         subset=args.subset_forcing,
+                                         refetch_records=False,
+                                         )
+
+# only run a specific chunck from the selection
+if args.global_chunk_number is not None:
+    if args.station_chunk_number is not None:
+        raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+
+
+    if not (int(args.split_by) > 0) :
+            raise ValueError("global_chunk_number is specified, but --split-by is not a strict positive number, so I don't know how to split the batch into chunks.")
+
+    run_station_chunk = None
+    print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+    totalchunks = 0
+    stations_iter = all_stations_select.iterrows()
+    in_current_chunk = False
+    try:
+        while not in_current_chunk:
+            istation,current_station = stations_iter.__next__()
+            all_records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+            chunks_current_station = math.ceil(float(len(all_records_morning_station_select))/float(args.split_by))
+            print('chunks_current_station',chunks_current_station)
+            in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+        
+            if in_current_chunk:
+                run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                run_station_chunk = int(args.global_chunk_number) - totalchunks 
+        
+            totalchunks +=chunks_current_station
+        
+
+    except StopIteration:
+       raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+    print("station = ",list(run_stations.index))
+    print("station chunk number:",run_station_chunk)
+
+# if no global chunk is specified, then run the whole station selection in one run, or
+# a specific chunk for each selected station according to # args.station_chunk_number
+else:
+    run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+    if args.station_chunk_number is not None:
+        run_station_chunk = int(args.station_chunk_number)
+        print("station(s) that is processed.",list(run_stations.index))
+        print("chunk number: ",run_station_chunk)
+    else:
+        if args.split_by != -1:
+            raise ValueError("Chunks are defined by --split-by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split-by.")
+        run_station_chunk = 0
+        print("stations that are processed.",list(run_stations.index))
+        
+
+#print(all_stations)
+print('Fetching initial/forcing records')
+records_morning = get_records(run_stations,\
+                              args.path_forcing,\
+                              subset=args.subset_forcing,
+                              refetch_records=False,
+                              )
+
+# note that if runtime is an integer number, we don't need to get the afternoon
+# profiles. 
+if args.runtime == 'from_afternoon_profile':
+    print('Fetching afternoon records for determining the simulation runtimes')
+    records_afternoon = get_records(run_stations,\
+                                    args.path_forcing,\
+                                    subset='afternoon',
+                                    refetch_records=False,
+                                    )
+    
+    # print(records_morning.index)
+    # print(records_afternoon.index)
+    # align afternoon records with the noon records, and set same index
+    print('hello')
+    print(len(records_afternoon))
+    print(len(records_morning))
+
+    print("aligning morning and afternoon records")
+    records_morning['dates'] = records_morning['ldatetime'].dt.date
+    records_afternoon['dates'] = records_afternoon['ldatetime'].dt.date
+    records_afternoon.set_index(['STNID','dates'],inplace=True)
+    ini_index_dates = records_morning.set_index(['STNID','dates']).index
+    records_afternoon = records_afternoon.loc[ini_index_dates]
+    records_afternoon.index = records_morning.index
+
+experiments = args.experiments.strip(' ').split(' ')
+for expname in experiments:
+    exp = EXP_DEFS[expname]
+    path_exp = args.path_experiments+'/'+expname+'/'
+
+    os.system('mkdir -p '+path_exp)
+    for istation,current_station in run_stations.iterrows():
+        print(istation,current_station)
+        records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+        if (int(args.split_by) * int(run_station_chunk)) >= (len(records_morning_station)):
+            print("warning: outside of profile number range for station "+\
+                  str(current_station)+". Skipping chunk number for this station.")
+        else:
+            fn_morning = args.path_forcing+'/'+format(current_station.name,'05d')+'_'+args.subset_forcing+'.yaml'
+            if os.path.isfile(fn_morning):
+                file_morning = open(fn_morning)
+            else:
+                fn_morning = \
+                     args.path_forcing+'/'+format(current_station.name,'05d')+\
+                     '_'+str(run_station_chunk)+'_'+args.subset_forcing+'.yaml'
+                file_morning = open(fn_morning)
+
+            if args.runtime == 'from_afternoon_profile':
+                file_afternoon = open(args.path_forcing+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+            fn_ini = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_ini.yaml'
+            fn_mod = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_mod.yaml'
+            file_ini = open(fn_ini,'w')
+            file_mod = open(fn_mod,'w')
+
+            #iexp = 0
+            onerun = False
+            print('starting station chunk number: '\
+                  +str(run_station_chunk)+'(size: '+str(args.split_by)+' soundings)')
+
+            records_morning_station_chunk = records_morning_station.iloc[((run_station_chunk)*int(args.split_by)):((run_station_chunk+1)*int(args.split_by))] #  [(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+
+            isim = 0
+            for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                    print('starting '+str(isim+1)+' out of '+\
+                      str(len(records_morning_station_chunk) )+\
+                      ' (station total: ',str(len(records_morning_station)),')')  
+                
+            
+                    c4gli_morning = get_record_yaml(file_morning, 
+                                                    record_morning.index_start, 
+                                                    record_morning.index_end,
+                                                    mode='ini')
+                    if args.diag_tropo is not None:
+                        print('add tropospheric parameters on advection and subsidence (for diagnosis)')
+                        seltropo = (c4gli_morning.air_ac.p > c4gli_morning.air_ac.p.iloc[-1]+ 3000.*(- 1.2 * 9.81 ))
+                        profile_tropo = c4gli_morning.air_ac[seltropo]
+                        for var in args.diag_tropo:#['t','q','u','v',]:
+                            if var[:3] == 'adv':
+                                mean_adv_tropo = np.mean(profile_tropo[var+'_x']+profile_tropo[var+'_y'] )
+                                c4gli_morning.update(source='era-interim',pars={var+'_tropo':mean_adv_tropo})
+                            else:
+                                print("warning: tropospheric variable "+var+" not recognized")
+                    
+                    
+                    if args.runtime == 'from_afternoon_profile':
+                        record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+                        c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                                          record_afternoon.index_start, 
+                                                          record_afternoon.index_end,
+                                                        mode='ini')
+                        runtime = int((c4gli_afternoon.pars.datetime_daylight - 
+                                             c4gli_morning.pars.datetime_daylight).total_seconds())
+                    elif args.runtime == 'from_input':
+                        runtime = c4gli_morning.pars.runtime
+                    else:
+                        runtime = int(args.runtime)
+
+            
+                    c4gli_morning.update(source='pairs',pars={'runtime' : \
+                                        runtime})
+                    c4gli_morning.update(source=expname, pars=exp)
+                    if expname[-5:] == '_VMAX':
+                        # ini_sel = ini[ini.cveg > ini.cveg.quantile(0.99)]
+                        # ini = c4gldata['GLOBAL_ADV'].frames['profiles']['records_all_stations_ini']
+                        c4gli_morning.update(source=expname, pars=\
+                                             {'alpha' : 0.19835934815210562,
+                                              'cveg' : 0.9324493037539419,
+                                              'z0m' : 1.287697822716918,
+                                              'z0h' : 0.1287697822716918,
+                                              'LAI' : 2.4429540235782645,
+                                              }\
+                                            )
+                    if expname[-3:] == '_V0':
+                        c4gli_morning.update(source=expname, pars=\
+                                             {'cveg': 0.0,
+                                             'z0m': 0.01,
+                                             'z0h': 0.001,
+                                             }\
+                                            )
+                    if expname[-5:] == '_VMIN':
+                        c4gli_morning.update(source=expname, pars=\
+                                             {'alpha': 0.2868081648656875,\
+                                             'cveg': 0.08275966502449594,\
+                                             'z0m': 0.05760000169277192,\
+                                             'z0h': 0.005760000169277192,\
+                                             'LAI': 2.0,\
+                                             }\
+                                            )
+                    if expname[-5:] == '_L001':
+                        #c4gldata['GLOBAL_ADV_SHR'].frames['stats']['records_all_stations_ini'].LAIpixel.quantile(0.01)
+
+                        c4gli_morning.update(source=expname, pars=\
+                                             {'LAI':0.111/0.325,\
+                                              'cveg':0.325,\
+                                              'z0m':0.17425,\
+                                              'z0h':0.017425,\
+                                              'alpha':0.270,\
+                                             }\
+                                            )
+                    if expname[-5:] == '_L099':
+                        # select = c4gldata['GLOBAL_ADV_SHR'].frames['stats']['records_all_stations_ini'].LAIpixel >= 4.38
+                        # np.mean(c4gldata['GLOBAL_ADV_SHR'].frames['stats']['records_all_stations_ini'][select].LAIpixel)
+                        # np.mean(c4gldata['GLOBAL_ADV_SHR'].frames['stats']['records_all_stations_ini'][select].LAIpixel)
+                        # np.mean(c4gldata['GLOBAL_ADV_SHR'].frames['stats']['records_all_stations_ini'][select].LAIpixel)
+                        c4gli_morning.update(source=expname, pars=\
+                                             {'LAI':4.605/0.807,\
+                                              'cveg':0.807,\
+                                              'z0m':1.78,\
+                                              'z0h':0.178,\
+                                              'alpha':0.195,\
+                                             }\
+                                            )
+
+                    c4gl = class4gl(c4gli_morning)
+
+                    if args.error_handling == 'dump_always':
+                        try:
+                            print('checking data sources')
+                            if not c4gli_morning.check_source_globaldata():
+                                print('Warning: some input sources appear invalid')
+                            c4gl.run()
+                            print('run succesful')
+                        except:
+                            print('run not succesful')
+                        onerun = True
+
+                        c4gli_morning.dump(file_ini)
+                        
+                        
+                        c4gl.dump(file_mod,\
+                                  include_input=False,\
+                                  #timeseries_only=timeseries_only,\
+                                 )
+                        onerun = True
+                    # in this case, only the file will dumped if the runs were
+                    # successful
+                    elif args.error_handling == 'dump_on_success':
+                       try:
+                            print('checking data sources')
+                            if not c4gli_morning.check_source_globaldata():
+                                print('Warning: some input sources appear invalid')
+                            c4gl.run()
+                            print('run succesfull')
+                            c4gli_morning.dump(file_ini)
+                            
+                            
+                            c4gl.dump(file_mod,\
+                                      include_input=False,\
+                                      #timeseries_only=timeseries_only,\
+                                     )
+                            onerun = True
+                       except:
+                           print('run not succesfull')
+                    isim += 1
+
+
+            file_ini.close()
+            file_mod.close()
+            file_morning.close()
+            if args.runtime == 'from_afternoon_profile':
+                file_afternoon.close()
+    
+            if onerun:
+                records_ini = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='ini',
+                                                           refetch_records=True,
+                                                           )
+                records_mod = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='mod',\
+                                                           refetch_records=True,\
+                                                           )
+            else:
+                # remove empty files
+                os.system('rm '+fn_ini)
+                os.system('rm '+fn_mod)
+    
+    # # align afternoon records with initial records, and set same index
+    # records_afternoon.index = records_afternoon.ldatetime.dt.date
+    # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+    # records_afternoon.index = records_ini.index
+    
+    # stations_for_iter = stations(path_exp)
+    # for STNID,station in stations_iterator(stations_for_iter):
+    #     records_current_station_index = \
+    #             (records_ini.index.get_level_values('STNID') == STNID)
+    #     file_current_station_mod = STNID
+    # 
+    #     with \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+    #     open(path_forcing+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+    #         for (STNID,index),record_ini in records_iterator(records_ini):
+    #             c4gli_ini = get_record_yaml(file_station_ini, 
+    #                                         record_ini.index_start, 
+    #                                         record_ini.index_end,
+    #                                         mode='ini')
+    #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+    # 
+    #             record_mod = records_mod.loc[(STNID,index)]
+    #             c4gl_mod = get_record_yaml(file_station_mod, 
+    #                                         record_mod.index_start, 
+    #                                         record_mod.index_end,
+    #                                         mode='mod')
+    #             record_afternoon = records_afternoon.loc[(STNID,index)]
+    #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+    #                                         record_afternoon.index_start, 
+    #                                         record_afternoon.index_end,
+    #                                         mode='ini')
+
diff --git a/class4gl/simulations/simulations_wwilt_wfc.py b/class4gl/simulations/simulations_wwilt_wfc.py
new file mode 100644
index 0000000..27a6d6a
--- /dev/null
+++ b/class4gl/simulations/simulations_wwilt_wfc.py
@@ -0,0 +1,381 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+#if __name__ == '__main__':
+parser = argparse.ArgumentParser()
+#parser.add_argument('--timestamp')
+parser.add_argument('--path_forcing')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/')
+parser.add_argument('--path_experiments')#,default='/user/data/gent/gvo000/gvo00090/D2D/data/C4GL/')
+parser.add_argument('--first_station_row')
+parser.add_argument('--last_station_row')
+parser.add_argument('--station_id') # run a specific station id
+parser.add_argument('--error_handling',default='dump_on_success')
+parser.add_argument('--diag_tropo',default=['advt','advq','advu','advv'])
+parser.add_argument('--subset_forcing',default='morning') # this tells which yaml subset
+                                                      # to initialize with.
+                                                      # Most common options are
+                                                      # 'morning' and 'ini'.
+
+# Tuntime is usually specified from the afternoon profile. You can also just
+# specify the simulation length in seconds
+parser.add_argument('--runtime',default='from_afternoon_profile')
+
+parser.add_argument('--experiments')
+parser.add_argument('--split_by',default=-1)# station soundings are split
+                                            # up in chunks
+
+#parser.add_argument('--station-chunk',default=0)
+parser.add_argument('--c4gl_path_lib')#,default='/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/lib')
+parser.add_argument('--global_chunk_number') # this is the batch number according to split-by in case of considering all stations
+parser.add_argument('--station_chunk_number') # this is the batch number according to split-by in case of considering all stations
+args = parser.parse_args()
+
+sys.path.insert(0, args.c4gl_path_lib)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+EXP_DEFS  =\
+{
+  'GLOBAL_NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC_WILT':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_NOAC_FC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_WILT':    {'sw_ac' : ['adv'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_ADV_FC':    {'sw_ac' : ['adv'],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'GLOBAL_AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+# ========================
+print("getting a list of stations")
+# ========================
+
+# these are all the stations that are found in the input dataset
+all_stations = stations(args.path_forcing,suffix=args.subset_forcing,refetch_stations=False)
+
+# ====================================
+print('defining all_stations_select')
+# ====================================
+
+# these are all the stations that are supposed to run by the whole batch (all
+# chunks). We narrow it down according to the station(s) specified.
+
+
+
+if args.station_id is not None:
+    print("Selecting station by ID")
+    stations_iter = stations_iterator(all_stations)
+    STNID,run_station = stations_iter.set_STNID(STNID=int(args.station_id))
+    all_stations_select = pd.DataFrame([run_station])
+else:
+    print("Selecting stations from a row range in the table")
+    all_stations_select = pd.DataFrame(all_stations.table)
+    if args.last_station_row is not None:
+        all_stations_select = all_station_select.iloc[:(int(args.last_station)+1)]
+    if args.first_station_row is not None:
+        all_stations_select = all_station_select.iloc[int(args.first_station):]
+print("station numbers included in the whole batch "+\
+      "(all chunks):",list(all_stations_select.index))
+
+print(all_stations_select)
+print("getting all records of the whole batch")
+all_records_morning_select = get_records(all_stations_select,\
+                                         args.path_forcing,\
+                                         subset=args.subset_forcing,
+                                         refetch_records=False,
+                                         )
+
+# only run a specific chunck from the selection
+if args.global_chunk_number is not None:
+    if args.station_chunk_number is not None:
+        raise ValueError('You need to specify either global-chunk-number or station-chunk-number, not both.')
+
+
+    if not (int(args.split_by) > 0) :
+            raise ValueError("global_chunk_number is specified, but --split-by is not a strict positive number, so I don't know how to split the batch into chunks.")
+
+    run_station_chunk = None
+    print('determining the station and its chunk number according global_chunk_number ('+args.global_chunk_number+')')
+    totalchunks = 0
+    stations_iter = all_stations_select.iterrows()
+    in_current_chunk = False
+    try:
+        while not in_current_chunk:
+            istation,current_station = stations_iter.__next__()
+            all_records_morning_station_select = all_records_morning_select.query('STNID == '+str(current_station.name))
+            chunks_current_station = math.ceil(float(len(all_records_morning_station_select))/float(args.split_by))
+            print('chunks_current_station',chunks_current_station)
+            in_current_chunk = (int(args.global_chunk_number) < (totalchunks+chunks_current_station))
+        
+            if in_current_chunk:
+                run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+                run_station_chunk = int(args.global_chunk_number) - totalchunks 
+        
+            totalchunks +=chunks_current_station
+        
+
+    except StopIteration:
+       raise ValueError("Could not determine station chunk number.  --global_chunk_number ("+args.global_chunk_number+") outside of range [0,"+ str(totalchunks)+'[')
+    print("station = ",list(run_stations.index))
+    print("station chunk number:",run_station_chunk)
+
+# if no global chunk is specified, then run the whole station selection in one run, or
+# a specific chunk for each selected station according to # args.station_chunk_number
+else:
+    run_stations = pd.DataFrame(all_stations_select)# run_stations.loc[(int(args.__dict__['last_station'])]
+    if args.station_chunk_number is not None:
+        run_station_chunk = int(args.station_chunk_number)
+        print("station(s) that is processed.",list(run_stations.index))
+        print("chunk number: ",run_station_chunk)
+    else:
+        if args.split_by != -1:
+            raise ValueError("Chunks are defined by --split-by, but I don't know which chunk to run. Please provide --global_chunk_number or --station_chunk_number, or leave out --split-by.")
+        run_station_chunk = 0
+        print("stations that are processed.",list(run_stations.index))
+        
+
+#print(all_stations)
+print('Fetching initial/forcing records')
+records_morning = get_records(run_stations,\
+                              args.path_forcing,\
+                              subset=args.subset_forcing,
+                              refetch_records=False,
+                              )
+
+# note that if runtime is an integer number, we don't need to get the afternoon
+# profiles. 
+if args.runtime == 'from_afternoon_profile':
+    print('Fetching afternoon records for determining the simulation runtimes')
+    records_afternoon = get_records(run_stations,\
+                                    args.path_forcing,\
+                                    subset='afternoon',
+                                    refetch_records=False,
+                                    )
+    
+    # print(records_morning.index)
+    # print(records_afternoon.index)
+    # align afternoon records with the noon records, and set same index
+    print('hello')
+    print(len(records_afternoon))
+    print(len(records_morning))
+
+    print("aligning morning and afternoon records")
+    records_morning['dates'] = records_morning['ldatetime'].dt.date
+    records_afternoon['dates'] = records_afternoon['ldatetime'].dt.date
+    records_afternoon.set_index(['STNID','dates'],inplace=True)
+    ini_index_dates = records_morning.set_index(['STNID','dates']).index
+    records_afternoon = records_afternoon.loc[ini_index_dates]
+    records_afternoon.index = records_morning.index
+
+experiments = args.experiments.strip(' ').split(' ')
+for expname in experiments:
+    exp = EXP_DEFS[expname]
+    path_exp = args.path_experiments+'/'+expname+'/'
+
+    os.system('mkdir -p '+path_exp)
+    for istation,current_station in run_stations.iterrows():
+        print(istation,current_station)
+        records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+        if (int(args.split_by) * int(run_station_chunk)) >= (len(records_morning_station)):
+            print("warning: outside of profile number range for station "+\
+                  str(current_station)+". Skipping chunk number for this station.")
+        else:
+            fn_morning = args.path_forcing+'/'+format(current_station.name,'05d')+'_'+args.subset_forcing+'.yaml'
+            if os.path.isfile(fn_morning):
+                file_morning = open(fn_morning)
+            else:
+                fn_morning = \
+                     args.path_forcing+'/'+format(current_station.name,'05d')+\
+                     '_'+str(run_station_chunk)+'_'+args.subset_forcing+'.yaml'
+                file_morning = open(fn_morning)
+
+            if args.runtime == 'from_afternoon_profile':
+                file_afternoon = open(args.path_forcing+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+            fn_ini = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_ini.yaml'
+            fn_mod = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_mod.yaml'
+            file_ini = open(fn_ini,'w')
+            file_mod = open(fn_mod,'w')
+
+            #iexp = 0
+            onerun = False
+            print('starting station chunk number: '\
+                  +str(run_station_chunk)+'(size: '+str(args.split_by)+' soundings)')
+
+            records_morning_station_chunk = records_morning_station.iloc[((run_station_chunk)*int(args.split_by)):((run_station_chunk+1)*int(args.split_by))] #  [(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+
+            isim = 0
+            for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                    print('starting '+str(isim+1)+' out of '+\
+                      str(len(records_morning_station_chunk) )+\
+                      ' (station total: ',str(len(records_morning_station)),')')  
+                
+            
+                    c4gli_morning = get_record_yaml(file_morning, 
+                                                    record_morning.index_start, 
+                                                    record_morning.index_end,
+                                                    mode='ini')
+                    if args.diag_tropo is not None:
+                        print('add tropospheric parameters on advection and subsidence (for diagnosis)')
+                        seltropo = (c4gli_morning.air_ac.p > c4gli_morning.air_ac.p.iloc[-1]+ 3000.*(- 1.2 * 9.81 ))
+                        profile_tropo = c4gli_morning.air_ac[seltropo]
+                        for var in args.diag_tropo:#['t','q','u','v',]:
+                            if var[:3] == 'adv':
+                                mean_adv_tropo = np.mean(profile_tropo[var+'_x']+profile_tropo[var+'_y'] )
+                                c4gli_morning.update(source='era-interim',pars={var+'_tropo':mean_adv_tropo})
+                            else:
+                                print("warning: tropospheric variable "+var+" not recognized")
+                    
+                    
+                    if args.runtime == 'from_afternoon_profile':
+                        record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+                        c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                                          record_afternoon.index_start, 
+                                                          record_afternoon.index_end,
+                                                        mode='ini')
+                        runtime = int((c4gli_afternoon.pars.datetime_daylight - 
+                                             c4gli_morning.pars.datetime_daylight).total_seconds())
+                    elif args.runtime == 'from_input':
+                        runtime = c4gli_morning.pars.runtime
+                    else:
+                        runtime = int(args.runtime)
+
+            
+                    c4gli_morning.update(source='pairs',pars={'runtime' : \
+                                        runtime})
+                    c4gli_morning.update(source=expname, pars=exp)
+                    if expname[-5:] == '_WILT':
+                        c4gli_morning.update(source=expname, pars=\
+                                             {'wg':c4gli_morning.pars.wwilt,\
+                                              'w2':c4gli_morning.pars.wwilt}\
+                                            )
+                    if expname[-3:] == '_FC':
+                        c4gli_morning.update(source=expname, pars=\
+                                             {'wg':c4gli_morning.pars.wfc,\
+                                              'w2':c4gli_morning.pars.wfc}\
+                                            )
+
+                    c4gl = class4gl(c4gli_morning)
+
+                    if args.error_handling == 'dump_always':
+                        try:
+                            print('checking data sources')
+                            if not c4gli_morning.check_source_globaldata():
+                                print('Warning: some input sources appear invalid')
+                            c4gl.run()
+                            print('run succesful')
+                        except:
+                            print('run not succesful')
+                        onerun = True
+
+                        c4gli_morning.dump(file_ini)
+                        
+                        
+                        c4gl.dump(file_mod,\
+                                  include_input=False,\
+                                  #timeseries_only=timeseries_only,\
+                                 )
+                        onerun = True
+                    # in this case, only the file will dumped if the runs were
+                    # successful
+                    elif args.error_handling == 'dump_on_success':
+                       try:
+                            print('checking data sources')
+                            if not c4gli_morning.check_source_globaldata():
+                                print('Warning: some input sources appear invalid')
+                            c4gl.run()
+                            print('run succesfull')
+                            c4gli_morning.dump(file_ini)
+                            
+                            
+                            c4gl.dump(file_mod,\
+                                      include_input=False,\
+                                      #timeseries_only=timeseries_only,\
+                                     )
+                            onerun = True
+                       except:
+                           print('run not succesfull')
+                    isim += 1
+
+
+            file_ini.close()
+            file_mod.close()
+            file_morning.close()
+            if args.runtime == 'from_afternoon_profile':
+                file_afternoon.close()
+    
+            if onerun:
+                records_ini = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='ini',
+                                                           refetch_records=True,
+                                                           )
+                records_mod = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='mod',\
+                                                           refetch_records=True,\
+                                                           )
+            else:
+                # remove empty files
+                os.system('rm '+fn_ini)
+                os.system('rm '+fn_mod)
+    
+    # # align afternoon records with initial records, and set same index
+    # records_afternoon.index = records_afternoon.ldatetime.dt.date
+    # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+    # records_afternoon.index = records_ini.index
+    
+    # stations_for_iter = stations(path_exp)
+    # for STNID,station in stations_iterator(stations_for_iter):
+    #     records_current_station_index = \
+    #             (records_ini.index.get_level_values('STNID') == STNID)
+    #     file_current_station_mod = STNID
+    # 
+    #     with \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+    #     open(path_forcing+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+    #         for (STNID,index),record_ini in records_iterator(records_ini):
+    #             c4gli_ini = get_record_yaml(file_station_ini, 
+    #                                         record_ini.index_start, 
+    #                                         record_ini.index_end,
+    #                                         mode='ini')
+    #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+    # 
+    #             record_mod = records_mod.loc[(STNID,index)]
+    #             c4gl_mod = get_record_yaml(file_station_mod, 
+    #                                         record_mod.index_start, 
+    #                                         record_mod.index_end,
+    #                                         mode='mod')
+    #             record_afternoon = records_afternoon.loc[(STNID,index)]
+    #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+    #                                         record_afternoon.index_start, 
+    #                                         record_afternoon.index_end,
+    #                                         mode='ini')
+
diff --git a/class4gl/simulations/trash/run_test.py b/class4gl/simulations/trash/run_test.py
new file mode 100644
index 0000000..767d960
--- /dev/null
+++ b/class4gl/simulations/trash/run_test.py
@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import io
+import os
+import numpy as np
+import datetime as dt
+import sys
+import pytz
+import math
+
+import argparse
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--global-chunk')
+    parser.add_argument('--first-station')
+    parser.add_argument('--last-station')
+    parser.add_argument('--dataset')
+    parser.add_argument('--path-soundings')
+    parser.add_argument('--experiments')
+    parser.add_argument('--split-by',default=-1)# station soundings are split
+                                                # up in chunks
+    parser.add_argument('--station-chunk')
+    parser.add_argument('--c4gl-path',default='')
+    args = parser.parse_args()
+
+if args.c4gl_path == '': 
+    sys.path.insert(0, '/user/data/gent/gvo000/gvo00090/D2D/software/CLASS/class4gl/')
+else:
+    sys.path.insert(0, args.c4gl_path)
+from class4gl import class4gl_input, data_global,class4gl
+from interface_multi import stations,stations_iterator, records_iterator,get_record_yaml,get_records
+from class4gl import blh,class4gl_input
+
+# this is a variant of global run in which the output of runs are still written
+# out even when the run crashes.
+
+# #only include the following timeseries in the model output
+# timeseries_only = \
+# ['Cm', 'Cs', 'G', 'H', 'L', 'LE', 'LEpot', 'LEref', 'LEsoil', 'LEveg', 'Lwin',
+#  'Lwout', 'Q', 'RH_h', 'Rib', 'Swin', 'Swout', 'T2m', 'dq', 'dtheta',
+#  'dthetav', 'du', 'dv', 'esat', 'gammaq', 'gammatheta', 'h', 'q', 'qsat',
+#  'qsurf', 'ra', 'rs', 'theta', 'thetav', 'time', 'u', 'u2m', 'ustar', 'uw',
+#  'v', 'v2m', 'vw', 'wq', 'wtheta', 'wthetae', 'wthetav', 'wthetae', 'zlcl']
+
+
+
+EXP_DEFS  =\
+{
+  'NOAC':    {'sw_ac' : [],'sw_ap': True,'sw_lit': False},
+  'ADV':{'sw_ac' : ['adv',],'sw_ap': True,'sw_lit': False},
+  'W':  {'sw_ac' : ['w',],'sw_ap': True,'sw_lit': False},
+  'AC': {'sw_ac' : ['adv','w'],'sw_ap': True,'sw_lit': False},
+}
+
+
+#SET = 'GLOBAL'
+SET = args.dataset
+
+if 'path-soundings' in args.__dict__.keys():
+    path_soundingsSET = args.__dict__['path-soundings']+'/'+SET+'/'
+else:
+    path_soundingsSET = '/user/data/gent/gvo000/gvo00090/D2D/data/SOUNDINGS/'+SET+'/'
+
+all_stations = stations(path_soundingsSET,suffix='morning',refetch_stations=True).table
+
+all_records_morning = get_records(all_stations,\
+                              path_soundingsSET,\
+                              subset='morning',
+                              refetch_records=False,
+                              )
+
+if args.global_chunk is not None:
+    totalchunks = 0
+    stations_iterator = all_stations.iterrows()
+    in_current_chunk = False
+    while not in_current_chunk:
+        istation,current_station = stations_iterator.__next__()
+        all_records_morning_station = all_records_morning.query('STNID == '+str(current_station.name))
+        chunks_current_station = math.ceil(float(len(all_records_morning_station))/float(args.split_by))
+        in_current_chunk = (int(args.global_chunk) < (totalchunks+chunks_current_station))
+
+        if in_current_chunk:
+            run_stations = pd.DataFrame([current_station])# run_stations.loc[(int(args.__dict__['last_station'])]
+            run_station_chunk = int(args.global_chunk) - totalchunks 
+
+        totalchunks +=chunks_current_station
+
+else:
+    run_stations = pd.DataFrame(all_stations)
+    if args.last_station is not None:
+        run_stations = run_stations.iloc[:(int(args.__dict__['last_station'])+1)]
+    if args.first_station is not None:
+        run_stations = run_stations.iloc[int(args.__dict__['first_station']):]
+    run_station_chunk = 0
+    if args.station_chunk is not None:
+        run_station_chunk = args.station_chunk
+
+#print(all_stations)
+print(run_stations)
+print(args.__dict__.keys())
+records_morning = get_records(run_stations,\
+                              path_soundingsSET,\
+                              subset='morning',
+                              refetch_records=False,
+                              )
+records_afternoon = get_records(run_stations,\
+                                path_soundingsSET,\
+                                subset='afternoon',
+                                refetch_records=False,
+                                )
+
+# align afternoon records with the noon records, and set same index
+records_afternoon.index = records_afternoon.ldatetime.dt.date
+records_afternoon = records_afternoon.loc[records_morning.ldatetime.dt.date]
+records_afternoon.index = records_morning.index
+
+experiments = args.experiments.split(';')
+
+for expname in experiments:
+    exp = EXP_DEFS[expname]
+    path_exp = '/kyukon/data/gent/gvo000/gvo00090/D2D/data/C4GL/'+SET+'_'+expname+'/'
+
+    os.system('mkdir -p '+path_exp)
+    for istation,current_station in run_stations.iterrows():
+        records_morning_station = records_morning.query('STNID == '+str(current_station.name))
+        if (int(args.split_by) * int(run_station_chunk)) >= (len(records_morning_station)):
+            print("warning: outside of profile number range for station "+\
+                  str(current_station)+". Skipping chunk number for this station.")
+        else:
+            file_morning = open(path_soundingsSET+'/'+format(current_station.name,'05d')+'_morning.yaml')
+            file_afternoon = open(path_soundingsSET+'/'+format(current_station.name,'05d')+'_afternoon.yaml')
+            fn_ini = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_ini.yaml'
+            fn_mod = path_exp+'/'+format(current_station.name,'05d')+'_'+\
+                     str(int(run_station_chunk))+'_mod.yaml'
+            file_ini = open(fn_ini,'w')
+            file_mod = open(fn_mod,'w')
+
+            #iexp = 0
+            onerun = False
+
+            records_morning_station_chunk = records_morning_station[(int(args.split_by)*run_station_chunk):(int(args.split_by)*(run_station_chunk+1))]
+            print(records_morning_station_chunk)
+
+            for (STNID,chunk,index),record_morning in records_morning_station_chunk.iterrows():
+                
+            
+                    c4gli_morning = get_record_yaml(file_morning, 
+                                                    record_morning.index_start, 
+                                                    record_morning.index_end,
+                                                    mode='ini')
+                    
+                    #print('c4gli_morning_ldatetime',c4gli_morning.pars.ldatetime)
+                    
+                    
+                    record_afternoon = records_afternoon.loc[(STNID,chunk,index)]
+                    c4gli_afternoon = get_record_yaml(file_afternoon, 
+                                                      record_afternoon.index_start, 
+                                                      record_afternoon.index_end,
+                                                    mode='ini')
+            
+                    c4gli_morning.update(source='pairs',pars={'runtime' : \
+                                        int((c4gli_afternoon.pars.datetime_daylight - 
+                                             c4gli_morning.pars.datetime_daylight).total_seconds())})
+                    c4gli_morning.update(source=expname, pars=exp)
+
+                    c4gl = class4gl(c4gli_morning)
+                    try:
+                        c4gl.run()
+                    except:
+                        print('run not succesfull')
+                    onerun = True
+
+                    c4gli_morning.dump(file_ini)
+                    
+                    
+                    c4gl.dump(file_mod,\
+                              include_input=False,\
+                              #timeseries_only=timeseries_only,\
+                             )
+                    onerun = True
+
+                #iexp = iexp +1
+            file_ini.close()
+            file_mod.close()
+            file_morning.close()
+            file_afternoon.close()
+    
+            if onerun:
+                records_ini = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='ini',
+                                                           refetch_records=True,
+                                                           )
+                records_mod = get_records(pd.DataFrame([current_station]),\
+                                                           path_exp,\
+                                                           getchunk = int(run_station_chunk),\
+                                                           subset='mod',\
+                                                           refetch_records=True,\
+                                                           )
+            else:
+                # remove empty files
+                os.system('rm '+fn_ini)
+                os.system('rm '+fn_mod)
+    
+    # # align afternoon records with initial records, and set same index
+    # records_afternoon.index = records_afternoon.ldatetime.dt.date
+    # records_afternoon = records_afternoon.loc[records_ini.ldatetime.dt.date]
+    # records_afternoon.index = records_ini.index
+    
+    # stations_for_iter = stations(path_exp)
+    # for STNID,station in stations_iterator(stations_for_iter):
+    #     records_current_station_index = \
+    #             (records_ini.index.get_level_values('STNID') == STNID)
+    #     file_current_station_mod = STNID
+    # 
+    #     with \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_ini.yaml','r') as file_station_ini, \
+    #     open(path_exp+'/'+format(STNID,"05d")+'_mod.yaml','r') as file_station_mod, \
+    #     open(path_soundings+'/'+format(STNID,"05d")+'_afternoon.yaml','r') as file_station_afternoon:
+    #         for (STNID,index),record_ini in records_iterator(records_ini):
+    #             c4gli_ini = get_record_yaml(file_station_ini, 
+    #                                         record_ini.index_start, 
+    #                                         record_ini.index_end,
+    #                                         mode='ini')
+    #             #print('c4gli_in_ldatetime 3',c4gli_ini.pars.ldatetime)
+    # 
+    #             record_mod = records_mod.loc[(STNID,index)]
+    #             c4gl_mod = get_record_yaml(file_station_mod, 
+    #                                         record_mod.index_start, 
+    #                                         record_mod.index_end,
+    #                                         mode='mod')
+    #             record_afternoon = records_afternoon.loc[(STNID,index)]
+    #             c4gl_afternoon = get_record_yaml(file_station_afternoon, 
+    #                                         record_afternoon.index_start, 
+    #                                         record_afternoon.index_end,
+    #                                         mode='ini')
+
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..22efb32
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,4 @@
+# Inside of setup.cfg
+[metadata]
+description-file = README.md
+
diff --git a/setup.py b/setup.py
index bfb44db..8bfffd9 100644
--- a/setup.py
+++ b/setup.py
@@ -1,12 +1,35 @@
-# build with "python setup.py build_ext --inplace"
 from distutils.core import setup
-from distutils.extension import Extension
-from Cython.Build import cythonize
-import numpy as np
-import os
+from setuptools import find_packages
 
-os.environ["CC"] = "g++-7"
 
+# I followed this tutorial to have both the git repository matched with the pip
+# repository: https://medium.com/@joel.barmettler/how-to-upload-your-python-package-to-pypi-65edc5fe9c56
 setup(
-    ext_modules = cythonize((Extension("ribtol", sources=["ribtol.pyx"], include_dirs=[np.get_include()], ), ))
+        name='class4gl',
+        version='1.0.2',
+        license='gpl-3.0',        # https://help.github.com/articles/licensing-a-repository
+        description = 'a framework to investigate the dynamics of the atmospheric boundary layer weather balloons worldwide', # Give a short description
+        author = 'Hendrik Wouters',                        # Type in your name
+        author_email = 'hendrik.wouters@ugent.be',         # Type in your E-Mail
+        url = 'https://github.com/hendrikwout/class4gl',   # Provide either the link to your github or to your website
+        download_url ='https://github.com/hendrikwout/class4gl/archive/v0.1.tar.gz',
+        # I explain this later on
+        keywords = ['atmospheric boundary layer', 'weather balloons',
+                    'land--atmosphere interactions'],   # Keywords
+        # packages=['class4gl'],
+        install_package_data = True,
+        packages=find_packages("."),
+        install_requires=['beautifulsoup4','pyyaml','pysolar','basemap','xarray'],
+        # long_description=open('README.md').read(),
+        classifiers=[
+                'Development Status :: 4 - Beta',      # Chose either "3 - Alpha", "4
+                #'Intended Audience :: Atmospheric scientists',
+                #'Topic :: modelling of the atmospheric boundary layer',
+                # 'License :: gpl-3.0',   
+                'Programming Language :: Python :: 3',      
+                # 'Programming Language :: Python :: 3.4',
+                # 'Programming Language :: Python :: 3.5',
+                # 'Programming Language :: Python :: 3.6',
+                # 'Programming Language :: Python :: 3.7',
+              ],
 )