final edits for v 0.1

Author: pdevis

hypernets_processor/calibration/calibrate.py

@@ -38,6 +38,13 @@ def calibrate_l1a(self, measurandstring, dataset_l0, dataset_l0_bla, swir=False)
             self.context.logger.error("the measurandstring needs to be either 'radiance' or 'irradiance")
             exit()
 
+
+        if self.context.get_config_value("plot_l0"):
+            if swir:
+                dataset_l0.attrs['product_name'] = dataset_l0.attrs[
+                                                        'product_name']+"_SWIR"
+            self.plot.plot_scans_in_series("digital_number",dataset_l0)
+
         calibrate_function = self._measurement_function_factory.get_measurement_function(
             self.context.get_config_value("measurement_function_calibrate"))
         input_vars = calibrate_function.get_argument_names()

hypernets_processor/data_io/template_util.py

@@ -210,9 +210,8 @@ def propagate_values(target_ds, source_ds, exclude=None):
 
         # Propagate data
         for common_variable_name in common_variable_names:
-            if not "radiance" in common_variable_name and not "irradiance" in common_variable_name:
-                if target_ds[common_variable_name].shape == source_ds[common_variable_name].shape:
-                    target_ds[common_variable_name].values = source_ds[common_variable_name].values
+            if target_ds[common_variable_name].shape == source_ds[common_variable_name].shape:
+                target_ds[common_variable_name].values = source_ds[common_variable_name].values
 
     # todo - add method to propagate common unpopulated metadata
 

hypernets_processor/etc/processor.config

@@ -1,6 +1,7 @@
 [Processor]
 archive_directory: C:\Users\pdv\PycharmProjects\hypernets_processor\hypernets_processor\out\
 network: w
+version: 0.1
 
 [Wavelength_cal]
 mapping_vis_a: +1.6499700E+02
@@ -81,6 +82,7 @@ write_l2a: True
 
 plot_l0: True
 plot_l1a: True
+plot_l1a_diff: True
 plot_l1b: True
 plot_l1c: True
 plot_l2a: True

hypernets_processor/etc/processor_2.config

@@ -42,7 +42,7 @@ write_l1b: True
 write_l1c: True
 write_l2a: True
 
-plot_l0: False
+plot_l0: True
 plot_l1a: True
 plot_l1a_diff: True
 plot_l1b: True

hypernets_processor/interpolation/interpolate.py

@@ -60,13 +60,13 @@ def interpolate_l1c(self,dataset_l1b_rad,dataset_l1b_irr):
         dataset_l1c=self.templ.l1c_from_l1b_dataset(dataset_l1b_rad)
         dataset_l1c["acquisition_time"].values = dataset_l1b_rad["acquisition_time"].values
 
-        dataset_l1c["radiance"].values = dataset_l1b_rad["radiance"].values
-        dataset_l1c["u_random_radiance"].values = dataset_l1b_rad["u_random_radiance"].values
-        dataset_l1c["u_systematic_indep_radiance"].values = dataset_l1b_rad["u_systematic_indep_radiance"].values
-        dataset_l1c["u_systematic_corr_rad_irr_radiance"].values = dataset_l1b_rad["u_systematic_corr_rad_irr_radiance"].values
-        dataset_l1c["corr_random_radiance"].values = dataset_l1b_rad["corr_random_radiance"].values
-        dataset_l1c["corr_systematic_indep_radiance"].values = dataset_l1b_rad["corr_systematic_indep_radiance"].values
-        dataset_l1c["corr_systematic_corr_rad_irr_radiance"].values = dataset_l1b_rad["corr_systematic_corr_rad_irr_radiance"].values
+        # dataset_l1c["radiance"].values = dataset_l1b_rad["radiance"].values
+        # dataset_l1c["u_random_radiance"].values = dataset_l1b_rad["u_random_radiance"].values
+        # dataset_l1c["u_systematic_indep_radiance"].values = dataset_l1b_rad["u_systematic_indep_radiance"].values
+        # dataset_l1c["u_systematic_corr_rad_irr_radiance"].values = dataset_l1b_rad["u_systematic_corr_rad_irr_radiance"].values
+        # dataset_l1c["corr_random_radiance"].values = dataset_l1b_rad["corr_random_radiance"].values
+        # dataset_l1c["corr_systematic_indep_radiance"].values = dataset_l1b_rad["corr_systematic_indep_radiance"].values
+        # dataset_l1c["corr_systematic_corr_rad_irr_radiance"].values = dataset_l1b_rad["corr_systematic_corr_rad_irr_radiance"].values
 
         dataset_l1c=self.interpolate_irradiance(dataset_l1c,dataset_l1b_irr)
 
@@ -161,7 +161,6 @@ def process_measurement_function(self,measurandstring,dataset,measurement_functi
         #     if u_systematic_input_quantities_corr[i] is not None:
         #         if len(u_systematic_input_quantities_corr[i].shape) < len(datashape):
         #             u_systematic_input_quantities_corr[i] = np.tile(u_systematic_input_quantities_corr[i], (datashape[1], 1)).T
-        print(*input_quantities)
         measurand = measurement_function(*input_quantities)
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
@@ -170,15 +169,14 @@ def process_measurement_function(self,measurandstring,dataset,measurement_functi
                                                             u_random_input_quantities,
                                                         param_fixed=[False,True,True],
                                                         repeat_dims=1)
-            print("shapes", cov_systematic_input_quantities_indep[2].shape,input_quantities[2].shape,u_systematic_input_quantities_indep[2].shape)
             u_syst_measurand_indep,corr_syst_measurand_indep = self.prop.propagate_systematic(
                 measurement_function,input_quantities,
                 u_systematic_input_quantities_indep,
                 cov_x=['rand']*len(u_random_input_quantities),return_corr=True,
                 corr_axis=0,param_fixed=[False,True,True],repeat_dims=1)
             u_syst_measurand_corr,corr_syst_measurand_corr = self.prop.propagate_systematic(
                 measurement_function,input_quantities,u_systematic_input_quantities_corr,
-                cov_x=['rand']*len(u_random_input_quantities),return_corr=True,
+                cov_x= ['rand']*len(u_random_input_quantities),return_corr=True,
                 corr_axis=0,param_fixed=[False,True,True],repeat_dims=1)
         dataset[measurandstring].values = measurand
         dataset["u_random_"+measurandstring].values = u_random_measurand

hypernets_processor/interpolation/measurement_functions/land_network_interpolation_irradiance_linear.py

@@ -20,15 +20,16 @@ def function(self,output_time,times_irr,irradiances):
                     irradiance_intfunc = scipy.interpolate.interp1d(times_irr,
                                                                     irradiances)
                     out[:,i]= irradiance_intfunc(output_time[i])
-                return out
+
         else:
             if output_time > max(times_irr):
-                return irradiances[:,times_irr == max(times_irr)][:,0]
+                out =  irradiances[:,times_irr == max(times_irr)][:,0]
             elif output_time<min(times_irr):
-                return irradiances[:,times_irr==min(times_irr)][:,0]
+                out =  irradiances[:,times_irr==min(times_irr)][:,0]
             else:
                 irradiance_intfunc=scipy.interpolate.interp1d(times_irr,irradiances)
-                return irradiance_intfunc(output_time)
+                out =  irradiance_intfunc(output_time)
+        return out
 
 
     @staticmethod

hypernets_processor/interpolation/measurement_functions/water_network_interpolation.py

@@ -20,15 +20,15 @@ def function(self,output_time,times,variables):
                     irradiance_intfunc = scipy.interpolate.interp1d(times,
                                                                     variables)
                     out[:,i] = irradiance_intfunc(output_time[i])
-                return out
         else:
             if output_time > max(times):
-                return variables[:,times == max(times)][:,0]
+                out= variables[:,times == max(times)]
             elif output_time < min(times):
-                return variables[:,times == min(times)][:,0]
+                out=  variables[:,times == min(times)]
             else:
                 irradiance_intfunc = scipy.interpolate.interp1d(times,variables)
-                return irradiance_intfunc(output_time)
+                out = irradiance_intfunc(output_time)
+        return out
 
     @staticmethod
     def get_name():

hypernets_processor/main_processor.py

@@ -70,8 +70,11 @@ def run(self):
         seq_dir = server_dir + seq_id+"/"
 
         l0_irr, l0_rad, l0_bla = HypernetsReader(self.context).read_sequence(seq_dir)
-        l0_bla["digital_number"].values=l0_bla["digital_number"].values/10.
-
+        #l0_bla["digital_number"].values=l0_bla["digital_number"].values/10.
+        l0_irr["integration_time"].values=1024*np.ones(len(l0_irr["integration_time"]))
+        l0_rad["integration_time"].values = 1024*np.ones(
+            len(l0_rad["integration_time"]))
+        l0_bla["integration_time"].values=1024*np.ones(len(l0_bla["integration_time"]))
         FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/")
         seq_id = os.path.basename(os.path.normpath(seq_dir)).replace("SEQ", "")
         print(seq_id)
@@ -192,6 +195,8 @@ def run(self):
         # L1b = rhymer.get_epsilon(L1b)
         L1b=rhymer.process_l1b(L1a_rad, L1a_irr)
         #
+        print(L1b["corr_systematic_corr_rad_irr_irradiance"])
+        panic
         L1c=rhymer.process_l1c(L1b)
         #L1d_irr = cal.average_l1b("irradiance", L1c)
         L1d= surf.process_l1d(L1c)

hypernets_processor/main_processor_outdoortest.py

@@ -202,10 +202,10 @@ def run(self):
             #     sequence_path)
             # self.context.logger.debug("Done")
 
-            l0_rad["digital_number"].values[:,0]=l0_rad["digital_number"].values[:,0]/1.5
-            l0_irr["digital_number"].values[:,0]=l0_irr["digital_number"].values[:,0]/1.5
-            l0_swir_rad["digital_number"].values[:,0]=l0_swir_rad["digital_number"].values[:,0]/1.5
-            l0_swir_irr["digital_number"].values[:,0]=l0_swir_irr["digital_number"].values[:,0]/1.5
+            l0_rad["digital_number"].values[:,0]=l0_rad["digital_number"].values[:,0]/1.25
+            l0_irr["digital_number"].values[:,0]=l0_irr["digital_number"].values[:,0]/1.25
+            l0_swir_rad["digital_number"].values[:,0]=l0_swir_rad["digital_number"].values[:,0]/1.25
+            l0_swir_irr["digital_number"].values[:,0]=l0_swir_irr["digital_number"].values[:,0]/1.25
 
 
             #Calibrate to L1a

hypernets_processor/plotting/plotting.py

@@ -70,23 +70,23 @@ def plot_diff_scans(self,measurandstring,dataset,dataset_avg=None):
                        dataset.attrs['product_name']+"_series_"+str(
                 series_id[i])+"."+self.context.get_config_value("plotting_format"))
 
-            ids = np.where(dataset['series_id'] == series_id[i])
+            ids = np.where(dataset['series_id'] == series_id[i])[0]
 
             ydata_subset=dataset[measurandstring].values[:,ids]
             mask=DatasetUtil.unpack_flags(dataset["quality_flag"])["outliers"]
+            mask = mask[ids]
+
             if dataset_avg is None:
                 ids_used = np.where((dataset['series_id'] == series_id[i]) & np.invert(
                     DatasetUtil.unpack_flags(dataset["quality_flag"])["outliers"]))[0]
                 ydata_subset_used = dataset[measurandstring].values[:,ids_used]
-                avgs=np.tile(np.mean(ydata_subset_used,axis=1)[...,None],len(ids_used))
-                mask=mask[ids_used]
+                avgs=np.tile(np.mean(ydata_subset_used,axis=1)[...,None],len(ids))
             else:
                 avg_ids = np.where((dataset_avg['series_id'] == series_id[i]))[0]
-                avgs = np.tile(dataset_avg[measurandstring].values[:,avg_ids],len(avg_ids))
-                mask=mask[avg_ids]
+                avgs = np.tile(dataset_avg[measurandstring].values[:,avg_ids],len(ids))
 
             self.plot_variable("relative difference",plotpath,dataset["wavelength"].values,
-                               (ydata_subset-avgs)/avgs,ylim=[-0.2,0.2],mask=mask)
+                               (ydata_subset-avgs)/avgs,ylim=[-0.3,0.3],mask=mask)
 
     def plot_radiance(self,plotpath,xdata,ydata,labels=None):
         fig1,ax1 = plt.subplots(figsize=(10,5))
@@ -99,7 +99,7 @@ def plot_radiance(self,plotpath,xdata,ydata,labels=None):
         ax1.set_xlabel("Wavelength (nm)")
         ax1.set_ylabel(r"Radiance ($mW\ nm^{-1}\ m^{-2}\ sr^{-1}$)")
         ymax = np.percentile(ydata,95)*1.2
-        ax1.set_ylim([0,max(1,ymax)])
+        ax1.set_ylim([0,ymax])
         fig1.savefig(plotpath,bbox_inches='tight')
         plt.close(fig1)
 
@@ -114,7 +114,7 @@ def plot_irradiance(self,plotpath,xdata,ydata,labels=None):
         ax1.set_xlabel("Wavelength (nm)")
         ax1.set_ylabel(r"Irradiance ($mW\ nm^{-1}\ m^{-2}$)")
         ymax = np.percentile(ydata,95)*1.2
-        ax1.set_ylim([0,max(100,ymax)])
+        ax1.set_ylim([0,ymax])
         fig1.savefig(plotpath,bbox_inches='tight')
         plt.close(fig1)
 
@@ -142,28 +142,32 @@ def plot_reflectance(self,plotpath,xdata,ydata,labels=None):
         ax1.set_xlabel("Wavelength (nm)")
         ax1.set_ylabel(r"Reflectance")
         ymax=np.percentile(ydata,95)*1.2
-        ax1.set_ylim([0,max(0.2,ymax)])
+        ax1.set_ylim([0,ymax])
         fig1.savefig(plotpath,bbox_inches='tight')
         plt.close(fig1)
 
     def plot_other_var(self,measurandstring,plotpath,xdata,ydata,labels=None,ylim=None,mask=None):
         fig1,ax1 = plt.subplots(figsize=(10,5))
-        if labels is None and mask is None or len(np.where(mask))==0:
+        if labels is None and mask is None or len(np.where(mask)[0])==0:
             ax1.plot(xdata,ydata,alpha=0.3)
         elif mask is None:
             for i in range(len(labels)):
                 ax1.plot(xdata,ydata[:,i],label=labels[i],alpha=0.3)
         else:
-            print(ydata.shape)
-            print(ydata[:,np.where(np.invert(mask))].shape,ydata[:,np.where(np.invert(mask))][:,0].shape)
-            ax1.plot(xdata,ydata[:,np.where(mask)].reshape((len(ydata),len(np.where(mask)[0]))),label="masked",alpha=0.3)
-            ax1.plot(xdata,ydata[:,np.where(np.invert(mask))].reshape((len(ydata),len(np.where(np.invert(mask))[0]))),label="used",alpha=0.3)
+            print(len(np.where(mask)[0]))
+            ax1.plot(xdata,ydata[:,np.where(mask)].reshape((len(ydata),len(np.where(mask)[0]))),label="masked",alpha=0.3,color="red")
+            ax1.plot(xdata,ydata[:,np.where(np.invert(mask))].reshape((len(ydata),len(np.where(np.invert(mask))[0]))),label="used",alpha=0.3,color="green")
 
         if labels is not None or mask is not None:
-            ax1.legend()
+            handles,labels = plt.gca().get_legend_handles_labels()
+            by_label = dict(zip(labels,handles))
+            ax1.legend(by_label.values(),by_label.keys())
         ax1.set_xlabel("Wavelength (nm)")
         ax1.set_ylabel(measurandstring)
         if ylim is not None:
             ax1.set_ylim(ylim)
+        else:
+            ymax = np.percentile(ydata,95)*1.2
+            ax1.set_ylim([0,ymax])
         fig1.savefig(plotpath,bbox_inches='tight')
         plt.close(fig1)