Merge pull request #56 from HYPERNETS/setup_cfg

Setup cfg

Author: shunt16

examples/create_sample_file.py

@@ -7,10 +7,10 @@
 from os.path import join as pjoin
 from hypernets_processor import HypernetsDSBuilder
 from hypernets_processor.data_io.hypernets_writer import HypernetsWriter
-from hypernets_processor.data_io.filename_util import FilenameUtil
+from hypernets_processor.data_io.product_name_util import ProductNameUtil
 
 
-'''___Authorship___'''
+"""___Authorship___"""
 __author__ = "Sam Hunt"
 __created__ = "12/2/2020"
 __version__ = "0.0"
@@ -24,11 +24,15 @@
 
 def create_sample_W_L1B_file(output_directory, n_sequence, site):
     dsb = HypernetsDSBuilder()
-    dataset = dsb.create_ds_template({"wavelength": N_WAVELENGTHS, "sequence": n_sequence}, "W_L1B")
+    dataset = dsb.create_ds_template(
+        {"wavelength": N_WAVELENGTHS, "sequence": n_sequence}, "W_L1B"
+    )
 
     # wavelength data
     # todo - fix issue with wavelength assignment as coordinate
-    dataset["wavelength"].data = np.concatenate((np.arange(400, 1000, 3), np.arange(1000, 1700+10, 10)))
+    dataset["wavelength"].data = np.concatenate(
+        (np.arange(400, 1000, 3), np.arange(1000, 1700 + 10, 10))
+    )
     dataset["bandwidth"].data = np.random.normal(1.0, 0.5, N_WAVELENGTHS)
 
     # geometry data
@@ -38,23 +42,25 @@ def create_sample_W_L1B_file(output_directory, n_sequence, site):
     dataset["solar_zenith_angle"].data = np.linspace(30, 60, n_sequence)
 
     # observation data
-    dataset["upwelling_radiance"].data = np.round(np.random.rand(N_WAVELENGTHS, n_sequence), 3)
+    dataset["upwelling_radiance"].data = np.round(
+        np.random.rand(N_WAVELENGTHS, n_sequence), 3
+    )
     # dataset["u_random_reflectance"].data = np.random.normal(1.0, 0.5, (N_WAVELENGTHS, n_sequence))
     # dataset["u_systematic_reflectance"].data = np.random.normal(1.0, 0.5, (N_WAVELENGTHS,n_sequence))
     # dataset["cov_random_reflectance"].data = np.random.normal(1.0, 0.5, (N_WAVELENGTHS, N_WAVELENGTHS))
     # dataset["cov_systematic_reflectance"].data = np.random.normal(1.0, 0.5, (N_WAVELENGTHS, N_WAVELENGTHS))
 
     # time data
-    dataset["acquisition_time"].data = np.arange(10000, 10000+n_sequence, dtype=int)
+    dataset["acquisition_time"].data = np.arange(10000, 10000 + n_sequence, dtype=int)
 
     # make file name
-    fu = FilenameUtil()
+    fu = ProductNameUtil()
     filename = fu.create_file_name_l1b("w", site, datetime.datetime.today(), "0.00")
 
     # write file
     hw = HypernetsWriter()
     hw.write(dataset, pjoin(output_directory, filename))
 
 
-if __name__ == '__main__':
-    create_sample_W_L1B_file(".", 26,  "BSBE")
+if __name__ == "__main__":
+    create_sample_W_L1B_file(".", 26, "BSBE")

hypernets_processor/context.py

@@ -6,108 +6,105 @@
 import dataset
 
 
-'''___Authorship___'''
+"""___Authorship___"""
 __author__ = "Sam Hunt"
 __created__ = "3/8/2020"
 __version__ = __version__
 __maintainer__ = "Sam Hunt"
 __email__ = "sam.hunt@npl.co.uk"
 __status__ = "Development"
 
+PROCESSOR_CONFIG_PROTECTED_VALUES = []
+
 
 class Context:
     """
     Class to determine and store processor state
+
+    :type processor_config: configparser.RawConfigParser
+    :param processor_config: processor config data object
+
+    :type job_config: configparser.RawConfigParser
+    :param job_config: job config data object
     """
 
-    def __init__(self, job_config=None, processor_config=None, logger=None):
+    def __init__(self, processor_config=None, job_config=None, logger=None):
 
         # Initialise attributes
-        self.logger = None
+        self.config_values = {}
+        self.logger = logger
         self.metadata_db = None
         self.anomoly_db = None
 
-        # File attributes
-        self.time = None
-
-        # Processor Attributes
-        # Processor
-        self.version = None
+        # Unpack processor_config to set relevant attributes
+        if processor_config is not None:
+            self.unpack_config(processor_config)
 
-        # Input
-        self.metadata_db_url = None
+        # Unpack processor_config to set relevant attributes
+        if job_config is not None:
+            self.unpack_config(
+                job_config, protected_values=PROCESSOR_CONFIG_PROTECTED_VALUES
+            )
 
-        # Output
-        self.raw_data_directory = None
+        # Connect to metadata database
+        if "metadata_db_url" in self.get_config_names():
+            self.metadata_db = dataset.connect(self.get_config_value("metadata_db_url"))
 
-        # Job attributes
-        # Info
-        self.network = None
-        self.site = None
+        # Connect to anomoly database
+        if "anomoly_db_url" in self.get_config_names():
+            self.anomoly_db = dataset.connect(self.get_config_value("anomoly_db_url"))
 
-        # Input
-        self.raw_data_directory = None
-        self.anomoly_db_url = None
+    def unpack_config(self, config, protected_values=None):
+        """
+        Unpacks config data, sets relevant entries to values instance attribute
 
-        # Processing Options
-        self.l1_mf_name = None
-        self.l2_mf_name = None
-        self.write_l1a = None
+        :type config: configparser.RawConfigParser
+        :param config: config data
+        """
 
-        # Set logger attribute
-        if logger is not None:
-            self.logger = logger
+        protected_values = [] if protected_values is None else protected_values
 
-        # Unpack job_config to set relevant attributes
-        if job_config is not None:
-            self._unpack_job_config(job_config)
+        for section in config.sections():
+            for name in config[section].keys():
 
-            # Connect to anomoly database
-            self.anomoly_db = dataset.connect(self.anomoly_db_url)
+                if name not in protected_values:
+                    value = get_config_value(config, section, name)
+                    self.set_config_value(name, value)
 
-        # Unpack processor_config to set relevant attributes
-        if processor_config is not None:
-            self._unpack_processor_config(processor_config)
+    def set_config_value(self, name, value):
+        """
+        Sets config data to values instance attribute
 
-            # Connect to metadata database
-            self.metadata_db = dataset.connect(self.metadata_db_url)
+        :type name: str
+        :param name: config data name
 
-    def _unpack_job_config(self, job_config):
+        :param value: config data value
         """
-        Unpacks processor_config, sets relevant object attributes
 
-        :type job_config: configparser.RawConfigParser
-        :param job_config: job configuration information
+        self.config_values[name] = value
 
+    def get_config_value(self, name):
         """
+        Get config value
 
-        # Unpack Info
-        self.network = get_config_value(job_config, "Info", "network")
-        self.site = get_config_value(job_config, "Info", "site")
+        :type name: str
+        :param name: config data name
 
-        # Unpack Input
-        self.raw_data_directory = get_config_value(job_config, "Input", "raw_data_directory")
-
-        # Unpack Output
-        self.anomoly_db_url = get_config_value(job_config, "Output", "anomoly_db_url")
+        :return: config value
+        """
 
-        # Unpack processing options
-        self.l1_mf_name = get_config_value(job_config, "Processing Options", "measurement_function_name")
-        self.l2_mf_name = get_config_value(job_config, "Processing Options", "reflectance_protocol_name")
-        self.write_l1a = get_config_value(job_config, "Processing Options", "write_l1a", dtype=bool)
+        return self.config_values[name] if name in self.get_config_names() else None
 
-    def _unpack_processor_config(self, processor_config):
+    def get_config_names(self):
         """
-        Unpacks processor_config, sets relevant object attributes
+        Get available config value names
 
-        :type processor_config: configparser.RawConfigParser
-        :param processor_config: processor configuration information
+        :return: config value names
+        :rtype: list
         """
 
-        self.version = get_config_value(processor_config, "Processor", "version")
-        self.metadata_db_url = get_config_value(processor_config, "Input", "metadata_db_url")
-        self.archive_directory = get_config_value(processor_config, "Output", "archive_directory")
+        return list(self.config_values.keys())
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     pass

hypernets_processor/data_io/hypernets_db_builder.py

@@ -7,7 +7,7 @@
 from hypernets_processor.data_io.format.databases import DB_DICT_DEFS
 
 
-'''___Authorship___'''
+"""___Authorship___"""
 __author__ = "Sam Hunt"
 __created__ = "24/7/2020"
 __version__ = __version__
@@ -47,5 +47,5 @@ def create_db_template(url, db_format, db_format_defs=DB_DICT_DEFS):
         return create_template_db(url, schema_dict=schema_dict, schema_sql=schema_sql)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     pass

hypernets_processor/data_io/hypernets_ds_builder.py

@@ -9,7 +9,7 @@
 from hypernets_processor.data_io.format.variables import VARIABLES_DICT_DEFS
 
 
-'''___Authorship___'''
+"""___Authorship___"""
 __author__ = "Sam Hunt"
 __created__ = "6/5/2020"
 __version__ = __version__
@@ -33,8 +33,12 @@ class HypernetsDSBuilder:
     :param metadata_defs: dictionary of metadata for each product format (default is Hypernets formats)
     """
 
-    def __init__(self, context=None, variables_dict_defs=VARIABLES_DICT_DEFS, metadata_defs=METADATA_DEFS):
-    
+    def __init__(
+        self,
+        context=None,
+        variables_dict_defs=VARIABLES_DICT_DEFS,
+        metadata_defs=METADATA_DEFS,
+    ):
         self.context = context
         self.variables_dict_defs = variables_dict_defs
         self.metadata_defs = metadata_defs
@@ -64,8 +68,12 @@ def create_ds_template(self, dim_sizes_dict, ds_format, propagate_ds=None):
         if ds_format in self.return_ds_formats():
             variables_dict = self.variables_dict_defs[ds_format]
         else:
-            raise NameError("Invalid format name: " + ds_format + " - must be one of " +
-                            str(self.variables_dict_defs.keys()))
+            raise NameError(
+                "Invalid format name: "
+                + ds_format
+                + " - must be one of "
+                + str(self.variables_dict_defs.keys())
+            )
 
         # Find metadata def
         metadata = {}
@@ -88,11 +96,17 @@ def create_ds_template(self, dim_sizes_dict, ds_format, propagate_ds=None):
                 metadata_db_query = {}
 
         # Set product_name metadata
-        pu = ProductNameUtil()
-        metadata["product_name"] = pu.create_product_name(ds_format, context=self.context)
-
-        return create_template_dataset(variables_dict, dim_sizes_dict, metadata=metadata, propagate_ds=propagate_ds,
-                                       metadata_db=metadata_db, metadata_db_query=metadata_db_query)
+        pu = ProductNameUtil(context=self.context)
+        metadata["product_name"] = pu.create_product_name(ds_format)
+
+        return create_template_dataset(
+            variables_dict,
+            dim_sizes_dict,
+            metadata=metadata,
+            propagate_ds=propagate_ds,
+            metadata_db=metadata_db,
+            metadata_db_query=metadata_db_query,
+        )
 
     def return_ds_formats(self):
         """
@@ -171,5 +185,5 @@ def create_empty_dim_sizes_dict(self, ds_format):
         return dim_sizes_dict
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     pass