update inline with recent changes

Author: shunt16

hypernets_processor/test/test_functions.py

@@ -7,6 +7,10 @@
 from hypernets_processor.context import Context
 from hypernets_processor.utils.config import read_config_file
 from hypernets_processor.utils.logging import configure_logging
+from hypernets_processor.utils.config import (
+    PROCESSOR_CONFIG_PATH,
+    JOB_CONFIG_TEMPLATE_PATH,
+)
 from hypernets_processor.utils.paths import relative_path
 import datetime
 import os
@@ -23,13 +27,6 @@
 __email__ = "sam.hunt@npl.co.uk"
 __status__ = "Development"
 
-this_directory = os.path.dirname(__file__)
-TEMPLATE_PROCESSOR_CONFIG_PATH = relative_path(
-    "../etc/processor.config", this_directory
-)
-TEMPLATE_JOB_CONFIG_PATH = relative_path(
-    "../etc/job_template.config", this_directory
-)
 
 TEST_DS_DIM_SIZES_W = {"wavelength": 271, "series": 3, "scan": 10, "sequence": 1}
 
@@ -68,7 +65,48 @@ def setup_test_anomoly_db(url):
     del db
 
 
-def setup_test_processor_config(archive_directory=None, metadata_db_url=None):
+def setup_test_archive_db(url):
+    """
+    Creates archive_db for testing, populated with test data
+
+    :type url: str
+    :param url: database url
+    """
+
+    db = HypernetsDBBuilder.create_db_template(url, "archive")
+
+    db["products"].insert(
+        dict(
+            product_name="new_product1",
+            raw_product_name="SEQ20200311T112230",
+            site="site",
+        )
+    )
+    db["products"].insert(
+        dict(
+            product_name="new_product2",
+            raw_product_name="SEQ20200311T112330",
+            site="site",
+        )
+    )
+    db["products"].insert(
+        dict(
+            product_name="new_product3",
+            raw_product_name="SEQ20200311T112430",
+            site="site",
+        )
+    )
+
+    db.commit()
+    del db
+
+
+def setup_test_processor_config(
+    archive_directory=None,
+    metadata_db_url=None,
+    archive_db_url=None,
+    anomoly_db_url=None,
+):
     """
     Creates processor_config for testing
 
@@ -78,18 +116,32 @@ def setup_test_processor_config(archive_directory=None, metadata_db_url=None):
     :type metadata_db_url: str
     :param metadata_db_url: (optional) metadata db url, set if provided else default value used
 
+    :type archive_db_url: str
+    :param archive_db_url: (optional) archive db url, set if provided else default value used
+
+    :type anomoly_db_url: str
+    :param anomoly_db_url: (optional) anomoly db url, set if provided else default value used
+
     :return: test processor configuration information
     :rtype: configparser.RawConfigParser
     """
 
-    processor_config = read_config_file(TEMPLATE_PROCESSOR_CONFIG_PATH)
+    processor_config = read_config_file(PROCESSOR_CONFIG_PATH)
 
     processor_config["Processor"]["version"] = "0.0"
 
-    processor_config["Input"]["metadata_db_url"] = (
+    processor_config["Databases"]["metadata_db_url"] = (
         metadata_db_url if metadata_db_url is not None else "sqlite:///metadata.db"
     )
 
+    processor_config["Databases"]["anomoly_db_url"] = (
+        anomoly_db_url if anomoly_db_url is not None else "sqlite:///anomoly.db"
+    )
+
+    processor_config["Databases"]["archive_db_url"] = (
+        archive_db_url if archive_db_url is not None else "sqlite:///archive.db"
+    )
+
     processor_config["Output"]["archive_directory"] = (
         archive_directory if archive_directory is not None else "out"
     )
@@ -99,36 +151,24 @@ def setup_test_processor_config(archive_directory=None, metadata_db_url=None):
     return processor_config
 
 
-def setup_test_job_config(raw_data_directory=None, anomoly_db_url=None):
+def setup_test_job_config(raw_data_directory=None):
     """
     Creates processor_config for testing
 
     :type raw_data_directory: str
     :param raw_data_directory: (optional) raw data directory, set if provided else default value used
 
-    :type anomoly_db_url: str
-    :param anomoly_db_url: (opitional) anomoly db url, set if provided
-
     :return: test job configuration information
     :rtype: configparser.RawConfigParser
     """
-    job_config = read_config_file(TEMPLATE_JOB_CONFIG_PATH)
+    job_config = read_config_file(JOB_CONFIG_TEMPLATE_PATH)
 
-    job_config["Info"]["network"] = "land"
-    job_config["Info"]["site"] = "site"
+    job_config["Job"]["network"] = "land"
+    job_config["Job"]["site"] = "site"
 
     job_config["Input"]["raw_data_directory"] = (
         raw_data_directory if raw_data_directory is not None else "data"
     )
-    job_config["Output"]["anomoly_db_url"] = (
-        anomoly_db_url if anomoly_db_url is not None else "sqlite:///anomoly.db"
-    )
-
-    job_config["Processing Options"][
-        "measurement_function_name"
-    ] = "standard_measurement_function"
-    job_config["Processing Options"]["reflectance_protocol_name"] = "standard_protocol"
-    job_config["Processing Options"]["write_l1a"] = "True"
 
     return job_config
 
@@ -149,23 +189,27 @@ def setup_test_context(
     archive_directory=None,
     anomoly_db_url=None,
     metadata_db_url=None,
+    archive_db_url=None,
     create_directories=False,
     create_dbs=False,
 ):
     """
     Creates context for testing
 
+    :type raw_data_directory: str
+    :param raw_data_directory: (optional) raw data directory, set if provided else default value used
+
     :type archive_directory: str
     :param archive_directory: (optional) data archive directory, set if provided else default value used
 
+    :type anomoly_db_url: str
+    :param anomoly_db_url: (opitional) anomoly db url, set if provided
+
     :type metadata_db_url: str
     :param metadata_db_url: (optional) metadata db url, set if provided else default value used
 
-    :type raw_data_directory: str
-    :param raw_data_directory: (optional) raw data directory, set if provided else default value used
-
-    :type anomoly_db_url: str
-    :param anomoly_db_url: (opitional) anomoly db url, set if provided
+    :type archive_db_url: str
+    :param archive_db_url: (opitional) archive db url, set if provided
 
     :type create_directories: bool
     :param create_directories: option to create test directories at specified paths (default: False)
@@ -178,32 +222,48 @@ def setup_test_context(
     """
 
     processor_config = setup_test_processor_config(
-        archive_directory=archive_directory, metadata_db_url=metadata_db_url
-    )
-    job_config = setup_test_job_config(
-        raw_data_directory=raw_data_directory, anomoly_db_url=anomoly_db_url
+        archive_directory=archive_directory,
+        metadata_db_url=metadata_db_url,
+        anomoly_db_url=anomoly_db_url,
+        archive_db_url=archive_db_url,
     )
+    job_config = setup_test_job_config(raw_data_directory=raw_data_directory)
+
     logger = setup_test_logger()
 
     if create_directories:
         os.makedirs(processor_config["Output"]["archive_directory"])
         os.makedirs(job_config["Input"]["raw_data_directory"])
+        os.makedirs(
+            os.path.join(
+                job_config["Input"]["raw_data_directory"], "SEQ20200311T112330"
+            )
+        )
+        os.makedirs(
+            os.path.join(
+                job_config["Input"]["raw_data_directory"], "SEQ20200311T112430"
+            )
+        )
+        os.makedirs(
+            os.path.join(
+                job_config["Input"]["raw_data_directory"], "SEQ20200311T112530"
+            )
+        )
 
     if create_dbs:
-        setup_test_metadata_db(processor_config["Input"]["metadata_db_url"])
-        setup_test_anomoly_db(job_config["Output"]["anomoly_db_url"])
+        setup_test_metadata_db(processor_config["Databases"]["metadata_db_url"])
+        setup_test_anomoly_db(processor_config["Databases"]["anomoly_db_url"])
+        setup_test_archive_db(processor_config["Databases"]["archive_db_url"])
+
+    if not create_dbs:
+        processor_config["Databases"]["metadata_db_url"] = None
+        processor_config["Databases"]["anomoly_db_url"] = None
+        processor_config["Databases"]["archive_db_url"] = None
 
     context = Context(
         processor_config=processor_config, job_config=job_config, logger=logger
     )
 
-    if not create_dbs:
-        del context.metadata_db
-        context.metadata_db = None
-
-        del context.anomoly_db
-        context.anomoly_db = None
-
     context.set_config_value("time", datetime.datetime(2021, 4, 3, 11, 21, 15))
 
     return context
@@ -232,9 +292,13 @@ def teardown_test_context(context, remove_directories=False, remove_dbs=False):
         del context.anomoly_db
         os.remove(anomoly_db_path)
 
+        archive_db_path = context.archive_db.engine.url.database
+        del context.archive_db
+        os.remove(archive_db_path)
+
     if remove_directories:
-        shutil.rmtree(context.archive_directory)
-        shutil.rmtree(context.raw_data_directory)
+        shutil.rmtree(context.get_config_value("archive_directory"))
+        shutil.rmtree(context.get_config_value("raw_data_directory"))
 
 
 def create_test_ds(ds_format):