Merge the different branches and make them consistent

numerous changes to config files and formatting edits throughout the processor

Author: pdevis

hypernets_processor/calibration/calibrate.py

@@ -11,6 +11,7 @@
 from hypernets_processor.data_io.dataset_util import DatasetUtil
 from hypernets_processor.plotting.plotting import Plotting
 import numpy as np
+import os
 
 '''___Authorship___'''
 __author__ = "Pieter De Vis"
@@ -32,15 +33,15 @@ def __init__(self, context, MCsteps=1000, parallel_cores=0):
     def calibrate_l1a(self, measurandstring, dataset_l0, dataset_l0_bla):
 
         if measurandstring != "radiance" and measurandstring != "irradiance":
-            print("the measurandstring needs to be either 'radiance' or 'irradiance")
+            self.context.logger.error("the measurandstring needs to be either 'radiance' or 'irradiance")
             exit()
 
         calibrate_function = self._measurement_function_factory.get_measurement_function(
             self.context.get_config_value("measurement_function_calibrate"))
         input_vars = calibrate_function.get_argument_names()
 
+        dataset_l0 = self.preprocess_l0(dataset_l0)
         dataset_l1a = self.l1a_template_from_l0_dataset(measurandstring, dataset_l0)
-        dataset_l0, dataset_l1a = self.preprocess_l0(dataset_l0, dataset_l1a)
 
         calibration_data = self.prepare_calibration_data(measurandstring)
 
@@ -72,16 +73,16 @@ def calibrate_l1a(self, measurandstring, dataset_l0, dataset_l0_bla):
     def prepare_calibration_data(self,measurandstring):
         hypstar=self.context.get_config_value("hypstar_cal_number")
         caldate=self.context.get_config_value("cal_date")
-        directory=self.context.get_config_value("processor_directory")
-        non_linear_cals = np.genfromtxt(directory+r"/../examples/calibration_files/hypstar_"+
-                                str(hypstar)+"_nonlin_corr_coefs_"+str(caldate)+".dat")
+        directory=self.context.get_config_value("calibration_directory")
+        non_linear_cals = np.genfromtxt(os.path.join(directory,"hypstar_"+
+                                str(hypstar)+"_nonlin_corr_coefs_"+str(caldate)+".dat"))
 
         if measurandstring == "radiance":
-            gains = np.genfromtxt(directory+r"/../examples/calibration_files/hypstar_"+
-                                  str(hypstar)+"_radcal_L_"+str(caldate)+".dat")
+            gains = np.genfromtxt(os.path.join(directory,"hypstar_"+
+                                  str(hypstar)+"_radcal_L_"+str(caldate)+".dat"))
         else:
-            gains = np.genfromtxt(directory+r"/../examples/calibration_files/hypstar_"+
-                                  str(hypstar)+"_radcal_E_"+str(caldate)+".dat")
+            gains = np.genfromtxt(os.path.join(directory,"hypstar_"+
+                                  str(hypstar)+"_radcal_E_"+str(caldate)+".dat"))
 
         # print(non_linear_cals)
         # print(gains[:,0])
@@ -128,7 +129,8 @@ def calc_mean_masked(self, dataset, var, rand_unc=False, corr=False):
         else:
             out = np.empty((len(series_id), len(dataset['wavelength'])))
         for i in range(len(series_id)):
-            ids = np.where((dataset['series_id'] == series_id[i]) & (dataset['quality_flag'] == 0))
+            ids = np.where((dataset['series_id'] == series_id[i]) &
+                  np.invert(DatasetUtil.unpack_flags(dataset["quality_flag"])["outliers"]))
             out[i] = np.mean(dataset[var].values[:, ids], axis=2)[:, 0]
             if rand_unc:
                 out[i] = out[i] / len(ids[0])
@@ -223,7 +225,7 @@ def find_u_systematic_input(self, variables, dataset, datasetbla, ancillary_data
 
         return inputs
 
-    def preprocess_l0(self, datasetl0, datasetl1a):
+    def preprocess_l0(self, datasetl0):
         """
         Identifies and removes faulty measurements (e.g. due to cloud cover).
 
@@ -232,36 +234,13 @@ def preprocess_l0(self, datasetl0, datasetl1a):
         :return:
         :rtype:
         """
-        dim_sizes_dict = {"wavelength": len(datasetl0["wavelength"]), "scan": len(datasetl0["scan"])}
-        du = DatasetUtil()
-
         mask = self.clip_and_mask(datasetl0)
-        # datasetl0["quality_flag"].values = mask
-        # datasetl1a["quality_flag"].values = mask
-
-        flagval = 2 ** (self.context.get_config_value("outliers"))
-
-        print(np.where(mask > 0))
-
-        break
-        datasetl0["quality_flag"].values = [
-            flagval + datasetl0["quality_flag"].values[i] if mask[i] == 1 else
-            datasetl0["quality_flag"].values[i] for i in range(len(mask))]
-
 
+        datasetl0["quality_flag"][np.where(mask==1)] = DatasetUtil.set_flag(datasetl0["quality_flag"][np.where(mask==1)],"outliers") #for i in range(len(mask))]
 
-        datasetl0["quality_flag"].values = [
-            flagval + datasetl0["quality_flag"].values[i] if mask[i] == 1 else
-            datasetl0["quality_flag"].values[i] for i in range(len(mask))]
-
-
-        datasetl1a["quality_flag"].values = [
-            flagval + datasetl1a["quality_flag"].values[i] if mask[i] == 1 else
-            datasetl1a["quality_flag"].values[i] for i in range(len(mask))]
-
-        DN_rand = du.create_variable([len(datasetl0["wavelength"]), len(datasetl0["scan"])],
+        DN_rand = DatasetUtil.create_variable([len(datasetl0["wavelength"]), len(datasetl0["scan"])],
                                      dim_names=["wavelength", "scan"], dtype=np.uint32, fill_value=0)
-        DN_syst = du.create_variable([len(datasetl0["wavelength"]), len(datasetl0["scan"])],
+        DN_syst = DatasetUtil.create_variable([len(datasetl0["wavelength"]), len(datasetl0["scan"])],
                                      dim_names=["wavelength", "scan"], dtype=np.uint32, fill_value=0)
 
         datasetl0["u_random_digital_number"] = DN_rand
@@ -273,7 +252,7 @@ def preprocess_l0(self, datasetl0, datasetl1a):
         datasetl0["u_random_digital_number"].values = rand
         datasetl0["u_systematic_digital_number"] = DN_syst
 
-        return datasetl0, datasetl1a
+        return datasetl0
 
     def clip_and_mask(self, dataset, k_unc=3):
         mask = []
@@ -290,7 +269,6 @@ def clip_and_mask(self, dataset, k_unc=3):
             maski = np.zeros_like(intsig) # mask the columns that have NaN
             maski[np.where(np.abs(intsig - noiseavg) >= k_unc * noisestd)] = 1
             mask = np.append(mask, maski)
-            # print("mask",mask)
 
 
         # check if 10% of pixels are outiers
@@ -351,7 +329,9 @@ def l1a_template_from_l0_dataset(self, measurandstring, dataset_l0):
                                                        propagate_ds=dataset_l0)
         elif measurandstring == "irradiance":
             dataset_l1a = self.hdsb.create_ds_template(l1a_dim_sizes_dict, "L_L1A_IRR", propagate_ds=dataset_l0)
-        dataset_l1a["wavelength"]=dataset_l0["wavelength"]
+
+        dataset_l1a=dataset_l1a.assign_coords(wavelength=dataset_l0.wavelength)
+
         return dataset_l1a
 
     def l1b_template_from_l1a_dataset(self, measurandstring, dataset_l1a):
@@ -371,7 +351,7 @@ def l1b_template_from_l1a_dataset(self, measurandstring, dataset_l1a):
         elif measurandstring == "irradiance":
             dataset_l1b = self.hdsb.create_ds_template(l1b_dim_sizes_dict, "L_L1B_IRR", propagate_ds=dataset_l1a)
 
-        dataset_l1b["wavelength"].values = dataset_l1a["wavelength"].values
+        dataset_l1b=dataset_l1b.assign_coords(wavelength=dataset_l1a.wavelength)
 
         series_id = np.unique(dataset_l1a['series_id'])
         dataset_l1b["series_id"].values = series_id
@@ -380,7 +360,8 @@ def l1b_template_from_l1a_dataset(self, measurandstring, dataset_l1a):
                                "solar_azimuth_angle", "solar_zenith_angle"]:
             temp_arr = np.empty(len(series_id))
             for i in range(len(series_id)):
-                ids = np.where((dataset_l1a['series_id'] == series_id[i]) & (dataset_l1a['quality_flag'] == 0))
+                ids = np.where((dataset_l1a['series_id'] == series_id[i]) & np.invert(
+                     DatasetUtil.unpack_flags(dataset_l1a["quality_flag"])["outliers"]))
                 temp_arr[i] = np.mean(dataset_l1a[variablestring].values[ids])
             dataset_l1b[variablestring].values = temp_arr
 

hypernets_processor/cli/sequence_processor_cli.py

@@ -103,7 +103,7 @@ def cli():
                 if key[:5] == "write":
                     job_config["Output"][key] = "True"
 
-        job_config["Log"]["log_path"] = "test.log"#os.path.abspath(parsed_args.log) if parsed_args.log is not None else ""
+        job_config["Log"]["log_path"] = os.path.abspath(parsed_args.log) if parsed_args.log is not None else ""
         job_config["Log"]["verbose"] = str(parsed_args.verbose) if parsed_args.verbose is not None else ""
         job_config["Log"]["quiet"] = str(parsed_args.quiet) if parsed_args.verbose is not None else ""
 

hypernets_processor/context.py

@@ -22,10 +22,8 @@
 class Context:
     """
     Class to determine and store processor state
-
     :type processor_config: configparser.RawConfigParser
     :param processor_config: processor config data object
-
     :type job_config: configparser.RawConfigParser
     :param job_config: job config data object
     """
@@ -41,16 +39,12 @@ def __init__(self, processor_config=None, job_config=None, logger=None):
 
         # Unpack processor_config to set relevant attributes
         if processor_config is not None:
-            config_proc = configparser.ConfigParser()
-            config_proc.read(processor_config)
-            self.unpack_config(config_proc)
+            self.unpack_config(processor_config)
 
         # Unpack processor_config to set relevant attributes
         if job_config is not None:
-            config_job = configparser.ConfigParser()
-            config_job.read(job_config)
             self.unpack_config(
-                config_job, protected_values=PROCESSOR_CONFIG_PROTECTED_VALUES
+                job_config, protected_values=PROCESSOR_CONFIG_PROTECTED_VALUES
             )
 
         # Connect to databases
@@ -70,25 +64,23 @@ def __init__(self, processor_config=None, job_config=None, logger=None):
     def unpack_config(self, config, protected_values=None):
         """
         Unpacks config data, sets relevant entries to values instance attribute
-
         :type config: configparser.RawConfigParser
         :param config: config data
         """
 
         protected_values = [] if protected_values is None else protected_values
         for section in config.sections():
             for name in config[section].keys():
+
                 if name not in protected_values:
                     value = get_config_value(config, section, name)
                     self.set_config_value(name, value)
 
     def set_config_value(self, name, value):
         """
         Sets config data to values instance attribute
-
         :type name: str
         :param name: config data name
-
         :param value: config data value
         """
 
@@ -97,10 +89,8 @@ def set_config_value(self, name, value):
     def get_config_value(self, name):
         """
         Get config value
-
         :type name: str
         :param name: config data name
-
         :return: config value
         """
 
@@ -109,7 +99,6 @@ def get_config_value(self, name):
     def get_config_names(self):
         """
         Get available config value names
-
         :return: config value names
         :rtype: list
         """
@@ -118,4 +107,4 @@ def get_config_names(self):
 
 
 if __name__ == "__main__":
-    pass
+    pass

hypernets_processor/data_io/dataset_util.py

@@ -4,7 +4,7 @@
 
 from hypernets_processor.version import __version__
 import string
-from xarray import Variable, DataArray
+from xarray import Variable, DataArray, Dataset
 import numpy as np
 
 
@@ -35,7 +35,7 @@ def create_default_array(dim_sizes, dtype, dim_names=None, fill_value=None):
 
         :type dtype: type
         :param dtype: numpy data type
-dekrie
+
         :type dim_names: list
         :param dim_names: (optional) dimension names as strings, i.e. ["dim1_name", "dim2_name", "dim3_size"]
 

hypernets_processor/data_io/hypernets_reader.py

@@ -183,7 +183,7 @@ def read_series(self, seq_dir, series, lat, lon, metadata, flag, fileformat):
         # Header definition with length, description and decoding format
 
         header = self.read_header(f, HEADER_DEF)
-        self.context.logger.debugger(header)
+        self.context.logger.debug(header)
 
         pixCount = header['Pixel Count']
 
@@ -284,12 +284,13 @@ def read_series(self, seq_dir, series, lat, lon, metadata, flag, fileformat):
 
                 #             print(datetime.fromtimestamp(int(ts+timereboot)))
                 #             print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
-
-                ds.attrs["site_latitude"] = lat
-                ds.attrs["site_longitude"] = lon
-                ds["solar_zenith_angle"][scan_number] = get_altitude(float(lat), float(lon), acquisitionTime)
-                ds["solar_azimuth_angle"][scan_number] = get_azimuth(float(lat), float(lon), acquisitionTime)
-
+                if lat is not None:
+                    ds.attrs["site_latitude"] = lat
+                    ds.attrs["site_longitude"] = lon
+                    ds["solar_zenith_angle"][scan_number] = get_altitude(float(lat), float(lon), acquisitionTime)
+                    ds["solar_azimuth_angle"][scan_number] = get_azimuth(float(lat), float(lon), acquisitionTime)
+                else:
+                    self.context.logger.error("Lattitude is not found, using default values instead for lat, lon, sza and saa.")
                 ds['quality_flag'][scan_number] = flag
                 ds['integration_time'][scan_number] = header['integration_time']
                 ds['temperature'][scan_number] = header['temperature']
@@ -358,7 +359,7 @@ def read_metadata(self, seq_dir):
         #     ACTION_NONE  : 0x03   (03)
 
         metadata = ConfigParser()
-
+        print("seq",os.path.join(seq_dir, "metadata.txt"))
         if os.path.exists(os.path.join(seq_dir, "metadata.txt")):
             metadata.read(os.path.join(seq_dir, "metadata.txt"))
             # ------------------------------
@@ -455,7 +456,7 @@ def read_sequence(self, seq_dir, setfile=None):
             seq_dir)
 
         if seriesIrr:
-            l0_irr = self.read_series(seq_dir, seriesIrr, lat, lon, metadata, flag, "l0_irr")
+            l0_irr = self.read_series(seq_dir, seriesIrr, lat, lon, metadata, flag, "L0_IRR")
             if self.context.get_config_value("write_l0"):
                 self.writer.write(l0_irr, overwrite=True)
             # can't use this when non concatanted spectra
@@ -467,7 +468,7 @@ def read_sequence(self, seq_dir, setfile=None):
             self.context.logger.error("No irradiance data for this sequence")
 
         if seriesRad:
-            l0_rad = self.read_series(seq_dir, seriesRad, lat, lon, metadata, flag, "l0_rad")
+            l0_rad = self.read_series(seq_dir, seriesRad, lat, lon, metadata, flag, "L0_RAD")
             if self.context.get_config_value("write_l0"):
                 self.writer.write(l0_rad, overwrite=True)
         #         if all([os.path.isfile(os.path.join(seq_dir,"RADIOMETER/",f)) for f in seriesRad]):
@@ -478,7 +479,7 @@ def read_sequence(self, seq_dir, setfile=None):
             self.context.logger.error("No radiance data for this sequence")
 
         if seriesBlack:
-            l0_bla = self.read_series(seq_dir, seriesBlack, lat, lon, metadata, flag, "l0_bla")
+            l0_bla = self.read_series(seq_dir, seriesBlack, lat, lon, metadata, flag, "L0_BLA")
             if self.context.get_config_value("write_l0"):
                 self.writer.write(l0_bla, overwrite=True)
             # if all([os.path.isfile(os.path.join(seq_dir, "RADIOMETER/", f)) for f in seriesBlack]):