add operation to export part of the dataset to another file

Author: marconetto

src/hpcadvisor/__main__.py

@@ -109,6 +109,21 @@ def selecttask_handler(args):
     main_cli.main_selecttask(operation, userinput, taskfile, policy, numtasks)
 
 
+def datafilter_handler(args):
+
+    operation = args.operation
+    datafilter = args.datafilter
+    exportfile = args.exportfile
+
+    from hpcadvisor import main_cli
+
+    if operation == "export":
+        main_cli.main_datafilter(operation, datafilter, exportfile)
+    else:
+        print(f"Invalid operation: {operation}. Supported operations: export")
+        sys.exit(1)
+
+
 def _process_arguments():
     parent_parser = argparse.ArgumentParser(add_help=False)
     parent_parser.add_argument(
@@ -201,6 +216,12 @@ def _process_arguments():
     selecttask.add_argument("-n", "--numtasks", help="Number of tasks", required=False)
     selecttask.set_defaults(func=selecttask_handler)
 
+    datafilter = subparsers.add_parser("datafilter", help="Datafilter help")
+    datafilter.add_argument("operation", type=str)
+    datafilter.add_argument("-df", "--datafilter", help="Data filter", required=True)
+    datafilter.add_argument("-e", "--exportfile", help="Exported file", required=True)
+    datafilter.set_defaults(func=datafilter_handler)
+
     args = parser.parse_args()
 
     debug = args.debug

src/hpcadvisor/cli_data_filter.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+
+import itertools
+
+from hpcadvisor import dataset_handler, logger, price_puller, taskset_handler
+
+log = logger.logger
+
+
+def export(datafilter, exportfile):
+
+    datapoints = dataset_handler.get_datapoints(datafilter)
+    dataset_handler.store_datapoints(exportfile, datapoints)

src/hpcadvisor/dataset_handler.py

@@ -10,6 +10,12 @@
 datapoints_label = "datapoints"
 
 
+def store_datapoints(dataset_file, datapoints):
+
+    with open(dataset_file, "w") as outfile:
+        json.dump(datapoints, outfile, indent=2)
+
+
 def add_datapoint(dataset_file, datapoint):
     existing_data = {}
 
@@ -35,7 +41,9 @@ def get_plotfilter(plotfilter_file):
             if not isinstance(value, list):
                 plotfilter[key] = [value]
     else:
-        log.warning("Plotfilter file not provided or does not exist. Consuming entire dataset")
+        log.warning(
+            "Plotfilter file not provided or does not exist. Consuming entire dataset"
+        )
     return plotfilter
 
 
@@ -65,13 +73,18 @@ def is_valid_datapoint(datapoint, plotfilter):
         if key in datapoint:
             if key == "tags":
                 anymatch = False
-                if any(all(tag_name in datapoint[key] and datapoint[key][tag_name] == tag_value
-                               for tag_name, tag_value in values_dict.items())
-                           for values_dict in valuelist):
-                            anymatch = True
+                if any(
+                    all(
+                        tag_name in datapoint[key]
+                        and datapoint[key][tag_name] == tag_value
+                        for tag_name, tag_value in values_dict.items()
+                    )
+                    for values_dict in valuelist
+                ):
+                    anymatch = True
                 if not anymatch:
                     return False
-            elif  datapoint[key] not in valuelist:
+            elif datapoint[key] not in valuelist:
                 return False
         else:
             return False

src/hpcadvisor/main_cli.py

@@ -5,6 +5,7 @@
 from hpcadvisor import (
     batch_handler,
     cli_advice_generator,
+    cli_data_filter,
     cli_plot_generator,
     cli_task_selector,
     data_collector,
@@ -82,6 +83,11 @@ def main_advice(datafilter, appexectime):
     cli_advice_generator.generate_advice(datafilter, appexectime)
 
 
+def main_datafilter(operation, datafilter, exportfile):
+    log.info("Data filtering ...")
+    cli_data_filter.export(datafilter, exportfile)
+
+
 def main_selecttask(operation, userinput, taskfile, policy_name, num_tasks):
 
     if operation == "gettasks":