Merge pull request #82 from HYPERNETS/metadata_db

Metadata db

Author: shunt16

docs/sphinx/content/users/user_processor.rst

@@ -5,8 +5,8 @@
 
 .. _user_processor:
 
-Automated Processing User Guide
-===============================
+User Guide - Automated Processing
+=================================
 
 This section provides a user guide for running the `hypernets_processor` module as an automated processor of incoming field data. In this scenario, a set of field hypstar systems are regularly syncing raw data to a server.  Running on this server, the `hypernets_processor` processes the data and adds it to an archive that can be accessed through a user portal.
 
@@ -74,22 +74,22 @@ As well as defining required job configuration information, the job configuratio
 For all jobs, it is important relevant metadata be added to the metadata database, so it can be added to the data products.
 
 .. _user_processor-scheduler:
-Run Scheduler
--------------
+Running Job Scheduler
+---------------------
 
 Once setup the automated processing scheduler can be started with::
 
-   $ hypernets_processor_scheduler
+   $ hypernets_scheduler
 
 To see options, try::
 
-   $ hypernets_processor_scheduler --help
+   $ hypernets_scheduler --help
 
 All jobs are run regularly, processing any new data synced to the server from the field since the last run. The run schedule is defined in the scheduler config, which may be edited as::
 
    $ vim <installation_directory>/hypernets_processor/etc/scheduler.config
 
-Processed products are added to the data archive and listed in the archive database. Any anomolies are add to the anomoly database. More detailed job related log information is added to the job log file. Summary log information for all jobs is added to the processor log file.
+Processed products are added to the data archive and listed in the archive database. Any anomalies are add to the anomaly database. More detailed job related log information is added to the job log file. Summary log information for all jobs is added to the processor log file.
 
 To amend the list of scheduled jobs, edit the list of job configuration files listed in the processor jobs file as::
 

docs/sphinx/content/users/users.rst

@@ -8,9 +8,6 @@
 User Guide
 ==========
 
-Usage
------
-
 There are two main use cases for the hypernets_processor package. The primary function of the software is the automated preparation of data retrieved from network sites for distribution to users. Additionally, the software may also be used for ad-hoc processing of particular field acquisitions, for example for testing instrument operation in the field. For information on each these use cases click on one of the following links:
 
 

hypernets_processor/cli/hypernets_processor_cli.py

@@ -1,9 +1,9 @@
 """
-scheduler for hypernets_processor jobs cli
+cli for job scheduler
 """
 
 from hypernets_processor.version import __version__
-from hypernets_processor.cli.hypernets_scheduler_main import main
+from hypernets_processor.main.scheduler_main import main
 from hypernets_processor.utils.config import SCHEDULER_CONFIG_PATH
 import argparse
 
@@ -41,7 +41,7 @@ def configure_parser():
 
 def cli():
     """
-    Command line interface function for hypernets_scheduler_main
+    Command line interface function for scheduler_main
     """
 
     # run main

…processor/cli/hypernets_scheduler_cli.py hypernets_processor/cli/scheduler_cli.pyhypernets_processor/cli/hypernets_scheduler_cli.py renamed to hypernets_processor/cli/scheduler_cli.py hypernets_processor/cli/hypernets_scheduler_cli.py renamed to hypernets_processor/cli/scheduler_cli.py

@@ -0,0 +1,133 @@
+"""
+hypernets_processor cli
+"""
+
+from hypernets_processor.version import __version__
+from hypernets_processor.utils.config import (
+    PROCESSOR_CONFIG_PATH,
+    JOB_CONFIG_TEMPLATE_PATH,
+    PROCESSOR_WATER_DEFAULTS_CONFIG_PATH,
+    PROCESSOR_LAND_DEFAULTS_CONFIG_PATH
+)
+from hypernets_processor.utils.cli import configure_std_parser
+from hypernets_processor.utils.config import read_config_file
+from hypernets_processor.main.sequence_processor_main import main
+from datetime import datetime as dt
+import sys
+import os
+
+
+'''___Authorship___'''
+__author__ = "Sam Hunt"
+__created__ = "26/3/2020"
+__version__ = __version__
+__maintainer__ = "Sam Hunt"
+__email__ = "sam.hunt@npl.co.uk"
+__status__ = "Development"
+
+
+def configure_parser():
+    """
+    Configure parser
+
+    :return: parser
+    :rtype: argparse.ArgumentParser
+    """
+
+    description = "Tool for processing Hypernets Land and Water Network hyperspectral field data"
+
+    # Create standard parser
+    parser = configure_std_parser(description=description)
+
+    # Add specific arguments
+    parser.add_argument("-i", "--input-directory", action="store",
+                        help="Directory of input data")
+    parser.add_argument("-o", "--output-directory", action="store",
+                        help="Directory to write output data to")
+    parser.add_argument("-n", "--network", action="store", choices=["land", "water"],
+                        help="Network to process file for")
+    # parser.add_argument("--plot", action="store_true",
+    #                     help="Generate plots of processed data")
+    parser.add_argument("--write-all", action="store_true",
+                        help="Write all products at intermediate data processing levels before final product")
+    parser.add_argument("-j", "--job-config", action="store",
+                        help="Use instead of above arguments to specify job with configuration file")
+    return parser
+
+
+parser = configure_parser()
+parsed_args = parser.parse_args()
+
+
+if parsed_args.job_config and (
+        parsed_args.input_directory or parsed_args.output_directory
+        or parsed_args.network or parsed_args.write_all
+):
+    print("-j is mutually exclusive with other input arguments")
+    sys.exit(2)
+
+
+def cli():
+    """
+    Command line interface to sequence_processor_main for ad-hoc job processing
+    """
+
+    # If job config specified use
+    if parsed_args.job_config:
+        tmp_job = False
+        job_config_path = parsed_args.job_config
+
+    # Else build and write temporary job config from command line arguments
+    else:
+        tmp_job = True
+
+        processor_defaults = PROCESSOR_WATER_DEFAULTS_CONFIG_PATH
+        if parsed_args.network == "land":
+            processor_defaults = PROCESSOR_LAND_DEFAULTS_CONFIG_PATH
+
+        job_config = read_config_file([JOB_CONFIG_TEMPLATE_PATH, processor_defaults])
+
+        if parsed_args.input_directory is not None:
+            job_config["Input"]["raw_data_directory"] = os.path.abspath(parsed_args.input_directory)
+        else:
+            print("-i required")
+            sys.exit(2)
+
+        if parsed_args.output_directory is not None:
+            job_config["Output"]["archive_directory"] = os.path.abspath(parsed_args.output_directory)
+        else:
+            print("-o required")
+            sys.exit(2)
+
+        if parsed_args.write_all:
+            for key in job_config["Output"].keys():
+                if key[:5] == "write":
+                    job_config["Output"][key] = "True"
+
+        job_config["Log"]["log_path"] = os.path.abspath(parsed_args.log) if parsed_args.log is not None else ""
+        job_config["Log"]["verbose"] = str(parsed_args.verbose) if parsed_args.verbose is not None else ""
+        job_config["Log"]["quiet"] = str(parsed_args.quiet) if parsed_args.verbose is not None else ""
+
+        job_config["Job"]["job_name"] = "run_" + dt.now().strftime("%Y%m%dT%H%M%S")
+        home_directory = os.path.expanduser("~")
+        job_config["Job"]["job_working_directory"] = os.path.join(home_directory, ".hypernets", "tmp")
+        job_config_path = os.path.join(
+            job_config["Job"]["job_working_directory"],
+            job_config["Job"]["job_name"] + ".config"
+        )
+
+        os.makedirs(job_config["Job"]["job_working_directory"], exist_ok=True)
+        with open(job_config_path, "w") as f:
+            job_config.write(f)
+
+    # run main
+    main(processor_config_path=PROCESSOR_CONFIG_PATH, job_config_path=job_config_path, to_archive=False)
+
+    if tmp_job:
+        os.remove(job_config_path)
+
+    return None
+
+
+if __name__ == "__main__":
+    pass

hypernets_processor/cli/sequence_processor_cli.py

@@ -87,7 +87,7 @@ def cli():
             default="sqlite:///"+os.path.join(settings["working_directory"], db_fmt+".db"),
         )
 
-    settings["log_path"] = os.path.join(settings["working_directory"], "processor.log")
+    settings["log_path"] = os.path.join(settings["working_directory"], "scheduler.log")
 
     main(settings)
 

hypernets_processor/cli/setup_processor_cli.py

@@ -36,7 +36,7 @@ def __init__(self, processor_config=None, job_config=None, logger=None):
         self.config_values = {}
         self.logger = logger
         self.metadata_db = None
-        self.anomoly_db = None
+        self.anomaly_db = None
         self.archive_db = None
 
         # Unpack processor_config to set relevant attributes
@@ -49,7 +49,7 @@ def __init__(self, processor_config=None, job_config=None, logger=None):
                 job_config, protected_values=PROCESSOR_CONFIG_PROTECTED_VALUES
             )
 
-        # Connect to anomoly databases
+        # Connect to databases
         db_fmts = DB_DICT_DEFS.keys()
         for db_fmt in db_fmts:
             if db_fmt + "_db_url" in self.get_config_names():

hypernets_processor/context.py

@@ -40,8 +40,13 @@
 # Metadata Database
 METADATA_DB = {}
 
-# Anolomy Database
-ANOMOLY_DB = {}
+# Anomaly Database
+ANOMALY_DB = {"anomalies": {"columns": {"anomaly": {"type": str},
+                                        "raw_product_name": {"type": str},
+                                        "site": {"type": str}
+                                        }
+                            }
+              }
 
 # Archive Database
 ARCHIVE_DB = {"products": {"columns": {"product_name": {"type": str},
@@ -56,5 +61,5 @@
 # --------------------
 
 DB_DICT_DEFS = {"metadata": METADATA_DB,
-                "anomoly": ANOMOLY_DB,
+                "anomaly": ANOMALY_DB,
                 "archive": ARCHIVE_DB}