405 ux make config minimal

examples/sorting/plots.json

@@ -81,7 +81,8 @@
                                 "transformation": "performance",
                                 "xaxis": { "parameter": "elements", "label": "N" },
                                 "yaxis": { "label": "Execution time (s)" },
-                                "color_axis": { "parameter": "algorithm", "label": "Algorithm" }
+                                "color_axis": { "parameter": "algorithm", "label": "Algorithm" },
+                                "aggregations":[{"column":"perfvalue","agg":"filter:elapsed"}]
                             }
                         },
                         {

pyproject.toml

@@ -14,6 +14,7 @@ where = ["src"]
   'report/templates/**',
   'scripts/data/*',
   'scripts/data/website_images/*',
+  'scripts/data/supplemental-ui/**/*',
   'reframe/templates/**'
 ]
 

src/feelpp/benchmarking/dashboardRenderer/component/leaf.py

@@ -1,3 +1,4 @@
+import tempfile
 from feelpp.benchmarking.dashboardRenderer.component.base import GraphNode
 from feelpp.benchmarking.dashboardRenderer.repository.base import Repository
 from feelpp.benchmarking.dashboardRenderer.views.base import View
@@ -87,23 +88,34 @@ def patchTemplateInfo( self, patch : Union[dict,TemplateDataFile], prefix:str, s
             save (bool): If True, the patch will be written back to the associated data file
                          (e.g., a JSON file) on the filesystem.
         """
-        if save:
-            template_data_files = [d for d in self.view.template_info.data if isinstance(d,TemplateDataFile) and d.prefix and d.prefix == prefix ]
+        template_data_files = [d for d in self.view.template_info.data if isinstance(d,TemplateDataFile) and d.prefix and d.prefix == prefix ]
 
-            if len( template_data_files ) > 1:
-                warnings.warn(f"More than one file having prefix {prefix} found. First occurence will be overwritten")
+        if len( template_data_files ) > 1:
+            warnings.warn(f"More than one file having prefix {prefix} found. First occurence will be overwritten")
 
-            filepath = None
-            if len( template_data_files ) == 0:
-                warnings.warn(f"No data files with {prefix} found in {self.id}. Saving this patch will not be possible.")
+        patch_data = patch.model_dump() if hasattr(patch, "model_dump") else patch
+
+        if not template_data_files:
+            warnings.warn(f"No data files with {prefix} found in {self.id}. Saving/patching will not be possible.")
+        else:
+            target_file = template_data_files[0]
+            if save:
+                write_path = os.path.join(self.view.template_data_dir, target_file.filepath) if hasattr(self.view,"template_data_dir") and self.view.template_data_dir else target_file.filepath
             else:
-                filepath = template_data_files[0].filepath
-                format = template_data_files[0].format
-
-            if filepath:
-                with open( os.path.join( self.view.template_data_dir, filepath ), "w" ) as f:
-                    if format == "json":
-                        json.dump( patch.model_dump(), f )
-                    else:
-                        f.write( patch )
-        self.view.updateTemplateData( {prefix:patch} )
+                base_dir = self.view.template_data_dir if hasattr(self.view,"template_data_dir") and self.view.template_data_dir else (os.path.dirname(target_file.filepath) or ".")
+
+                tmp_fd, write_path = tempfile.mkstemp(dir=base_dir, suffix=f".{target_file.format}")
+                os.close(tmp_fd)
+            target_file.filepath = write_path
+
+            with open(write_path, "w") as f:
+                if target_file.format == "json":
+                    json.dump(patch_data, f)
+                else:
+                    f.write(patch_data)
+            self.view.updateTemplateData(target_file)
+
+            if not save: #Cleanup temp file
+                os.remove(write_path)
+
+        self.view.updateTemplateData( {prefix:patch_data} )

src/feelpp/benchmarking/dashboardRenderer/tests/test_node.py

@@ -1,13 +1,14 @@
 from feelpp.benchmarking.dashboardRenderer.component.base import GraphNode
 from feelpp.benchmarking.dashboardRenderer.component.leaf import LeafComponent
-from feelpp.benchmarking.dashboardRenderer.schemas.dashboardSchema import TemplateDataFile
+from feelpp.benchmarking.dashboardRenderer.schemas.dashboardSchema import TemplateDataFile, TemplateInfo
 
 import pytest
 
 class MockView:
     def __init__(self,name = ""):
         self.name = name
         self.template_data = {"test":"template_data"}
+        self.template_info = TemplateInfo(data=[TemplateDataFile(format="json",prefix="meta",filepath="")])
     def clone(self):
         return MockView(f"Cloned {self.name}")
     def updateTemplateData(self,data):
@@ -310,4 +311,4 @@ def test_leafcomponent_patchTemplateInfo_no_save():
     leaf.patchTemplateInfo(patch_data, prefix, save=False)
 
     assert view.template_data["meta"] == patch_data
-    # Ensure no file operations occurred (implicitly, as we aren't mocking open/json.dump)
+    # Ensure no file operations occurred (implicitly, as we aren't mocking open/json.dump)

src/feelpp/benchmarking/reframe/__main__.py

@@ -10,7 +10,12 @@
 def main_cli():
     parser = Parser()
 
-    machine_reader = ConfigReader(parser.args.machine_config,MachineConfig,"machine",dry_run=parser.args.dry_run)
+    if parser.args.machine_config:
+        machine_reader = ConfigReader(parser.args.machine_config,MachineConfig,"machine",dry_run=parser.args.dry_run)
+    else:
+        machine_reader = ConfigReader(None,MachineConfig,"machine",dry_run=parser.args.dry_run)
+        machine_reader.config = MachineConfig(machine="default")
+
 
     #Sets the cachedir and tmpdir directories for containers
     for platform, dirs in machine_reader.config.containers.items():
@@ -24,7 +29,8 @@ def main_cli():
 
     cmd_builder = CommandBuilder(machine_reader.config,parser)
 
-    os.environ["MACHINE_CONFIG_FILEPATH"] = parser.args.machine_config
+    if parser.args.machine_config:
+        os.environ["MACHINE_CONFIG_FILEPATH"] = parser.args.machine_config
 
     website_config = WebsiteConfigCreator(machine_reader.config.reports_base_dir)
 
@@ -38,7 +44,10 @@ def main_cli():
         app_reader = ConfigReader(configs,ConfigFile,"app",dry_run=parser.args.dry_run,additional_readers=[machine_reader])
 
         executable_name = os.path.basename(app_reader.config.executable).split(".")[0]
-        report_folder_path = cmd_builder.createReportFolder(executable_name,app_reader.config.use_case_name)
+        if parser.args.dry_run:
+            report_folder_path = None
+        else:
+            report_folder_path = cmd_builder.createReportFolder(executable_name,app_reader.config.use_case_name)
 
         #===============PULL IMAGES==================#
         if not parser.args.dry_run:
@@ -73,41 +82,43 @@ def main_cli():
         #================================================#
 
         #============== UPDATE WEBSITE CONFIG FILE ==============#
-        common_itempath = (parser.args.move_results or report_folder_path).split("/")
-        common_itempath = "/".join(common_itempath[:-1 - (common_itempath[-1] == "")])
+        if not parser.args.dry_run:
+            common_itempath = (parser.args.move_results or report_folder_path).split("/")
+            common_itempath = "/".join(common_itempath[:-1 - (common_itempath[-1] == "")])
 
-        website_config.updateExecutionMapping(
-            executable_name, machine_reader.config.machine, app_reader.config.use_case_name,
-            report_itempath = common_itempath
-        )
+            website_config.updateExecutionMapping(
+                executable_name, machine_reader.config.machine, app_reader.config.use_case_name,
+                report_itempath = common_itempath
+            )
 
-        website_config.updateMachine(machine_reader.config.machine)
-        website_config.updateUseCase(app_reader.config.use_case_name)
-        website_config.updateApplication(executable_name)
+            website_config.updateMachine(machine_reader.config.machine)
+            website_config.updateUseCase(app_reader.config.use_case_name)
+            website_config.updateApplication(executable_name)
 
-        website_config.save()
+            website_config.save()
         #======================================================#
 
 
         #============ CREATING RESULT ITEM ================#
-        with open(os.path.join(report_folder_path,"report.json"),"w") as f:
-            f.write(json.dumps(app_reader.config.json_report.model_dump()))
-
-        #Copy use case description if existant
-        FileHandler.copyResource(
-            app_reader.config.additional_files.description_filepath,
-            os.path.join(report_folder_path,"partials"),
-            "description"
-        )
+        if not parser.args.dry_run:
+            with open(os.path.join(report_folder_path,"report.json"),"w") as f:
+                f.write(json.dumps(app_reader.config.json_report.model_dump()))
+
+            #Copy use case description if existant
+            FileHandler.copyResource(
+                app_reader.config.additional_files.description_filepath,
+                os.path.join(report_folder_path,"partials"),
+                "description"
+            )
         #===============================================#
 
         try:
             # ============== LAUNCH REFRAME =======================#
-            reframe_cmd = cmd_builder.buildCommand( app_reader.config.timeout)
+            reframe_cmd = cmd_builder.buildCommand( app_reader.config.timeout )
             exit_code = subprocess.run(reframe_cmd, shell=True)
             #======================================================#
         finally:
-            if not os.path.exists(os.path.join(report_folder_path,"reframe_report.json")):
+            if report_folder_path and not os.path.exists(os.path.join(report_folder_path,"reframe_report.json")):
                 if os.path.exists(os.path.join(report_folder_path,"report.json")):
                     os.remove(os.path.join(report_folder_path,"report.json"))
                 os.rmdir(report_folder_path)
@@ -116,8 +127,9 @@ def main_cli():
         if parser.args.move_results:
             if not os.path.exists(parser.args.move_results):
                 os.makedirs(parser.args.move_results)
-            os.rename(os.path.join(report_folder_path,"reframe_report.json"),os.path.join(parser.args.move_results,"reframe_report.json"))
-            os.rename(os.path.join(report_folder_path,"report.json"),os.path.join(parser.args.move_results,"report.json"))
+            if report_folder_path:
+                os.rename(os.path.join(report_folder_path,"reframe_report.json"),os.path.join(parser.args.move_results,"reframe_report.json"))
+                os.rename(os.path.join(report_folder_path,"report.json"),os.path.join(parser.args.move_results,"report.json"))
         #======================================================#
 
     if parser.args.website:
@@ -126,4 +138,4 @@ def main_cli():
         subprocess.run(["npm","run","start"])
 
     # return exit_code.returncode
-    return 0
+    return 0

src/feelpp/benchmarking/reframe/commandBuilder.py

@@ -48,7 +48,6 @@ def buildJobOptions(self,timeout):
 
 
     def buildCommand(self,timeout):
-        assert self.report_folder_path is not None, "Report folder path not set"
         cmd = [
             'reframe',
             f'-C {self.buildConfigFilePath()}',
@@ -57,7 +56,7 @@ def buildCommand(self,timeout):
             f'--system={self.machine_config.machine}',
             f'--exec-policy={self.machine_config.execution_policy}',
             f'--prefix={self.machine_config.reframe_base_dir}',
-            f'--report-file={str(os.path.join(self.report_folder_path,"reframe_report.json"))}',
+            f'--report-file={str(os.path.join(self.report_folder_path,"reframe_report.json"))}' if self.report_folder_path else "",
             f"{self.buildJobOptions(timeout)}",
             f'--perflogdir=logs',
             f'{self.buildExecutionMode()}'

src/feelpp/benchmarking/reframe/config/configReader.py

@@ -118,8 +118,10 @@ def __init__(self, config_paths, schema, name, dry_run=False, additional_readers
         self.context = { "dry_run":dry_run }
         if config_paths:
             self.config = self.load( self.prepareConfigs(config_paths), schema )
+            self.original_config = self.config.model_copy()
+        else:
+            self.config = None
         self.name = name
-        self.original_config = self.config.model_copy()
         self.processor = TemplateProcessor()
         for additional_reader in additional_readers:
             self.updateConfig(TemplateProcessor.flattenDict(additional_reader.config,additional_reader.name))
@@ -169,6 +171,8 @@ def updateConfig(self, flattened_replace = None):
             flattened_replace: (dict) Containing all key, pair values that indicate the paths to replace. e.g { "replace.this.path": "with_this_value" }
                 If not provided, placeholders will be changed with own confing
         """
+        if not self.config:
+            return
         if not flattened_replace:
             flattened_replace = TemplateProcessor.flattenDict(self.config.model_dump())
         self.config = self.schema.model_validate(self.processor.recursiveReplace(self.config.model_dump(),flattened_replace), context=self.context)
@@ -180,4 +184,4 @@ def resetConfig(self, additional_readers = []):
         self.config = self.original_config.model_copy()
         for additional_reader in additional_readers:
             self.updateConfig(TemplateProcessor.flattenDict(additional_reader.config,additional_reader.name))
-        self.updateConfig()
+        self.updateConfig()

src/feelpp/benchmarking/reframe/config/machineConfigs/gaya/reframe.py

@@ -8,7 +8,7 @@
             'partitions': [
                 {
                     'name': 'production',
-                    'scheduler': 'squeue',
+                    'scheduler': 'slurm',
                     'launcher': 'srun',
                     'max_jobs': 8,
                     'access': ['--partition=production'],
@@ -37,7 +37,7 @@
                 },
                 {
                     'name': 'public',
-                    'scheduler': 'squeue',
+                    'scheduler': 'slurm',
                     'launcher': 'srun',
                     'max_jobs': 8,
                     'access': ['--partition=public'],
@@ -66,7 +66,7 @@
                 },
                 {
                     'name':'gpu',
-                    'scheduler':'squeue',
+                    'scheduler':'slurm',
                     'launcher':'srun',
                     'max_jobs':4,
                     'access': ['--partition=gpu'],

src/feelpp/benchmarking/reframe/parser.py

@@ -21,7 +21,7 @@ def processArgs(self):
     def addArgs(self):
         """ Add the necessary arguments to the parser"""
         options = self.parser.add_argument_group("Options")
-        options.add_argument('--machine-config', '-mc', required=True, type=str, metavar='MACHINE_CONFIG', help='Path to JSON reframe machine configuration file, specific to a system.')
+        options.add_argument('--machine-config', '-mc', required=False, default=None, type=str, metavar='MACHINE_CONFIG', help='Path to JSON reframe machine configuration file, specific to a system.')
         options.add_argument('--plots-config', '-pc', required=False, default=None, type=str, help='Path to JSON plots configuration file, used to generate figures. \nIf not provided, no plots will be generated. The plots configuration can also be included in the benchmark configuration file, under the "plots" field.')
         options.add_argument('--benchmark-config', '-bc', type=str, nargs='+', action='extend', default=[], metavar='CONFIG', help='Paths to JSON benchmark configuration files \nIn combination with --dir, specify only provide basenames for selecting JSON files.')
         options.add_argument('--custom-rfm-config', '-rc', type=str, required=False, default=None, help="Additional reframe configuration file to use instead of built-in ones. It should correspond the with the --machine-config specifications.")
@@ -40,7 +40,8 @@ def addArgs(self):
     def convertPathsToAbsolute(self):
         """ Converts arguments that contain paths to absolute. No change is made if absolute paths are provided"""
         self.args.benchmark_config = [os.path.abspath(c) for c in self.args.benchmark_config]
-        self.args.machine_config = os.path.abspath(self.args.machine_config)
+        if self.args.machine_config:
+            self.args.machine_config = os.path.abspath(self.args.machine_config)
         if self.args.plots_config:
             self.args.plots_config = os.path.abspath(self.args.plots_config)
 
@@ -54,10 +55,6 @@ def validate(self):
             print(f'[Error] --dir and --benchmark-config combination can only handle one DIR')
             sys.exit(1)
 
-        if not self.args.machine_config:
-            print(f'[Error] --machine-config should be specified')
-            sys.exit(1)
-
 
 
     def checkDirectoriesExist(self):
@@ -104,4 +101,4 @@ def listFilesAndExit(self):
         for config_path in self.args.benchmark_config:
             print(f"\t> {config_path}")
         print(f"\nTotal: {len(self.args.benchmark_config)} file(s)")
-        sys.exit(0)
+        sys.exit(0)

src/feelpp/benchmarking/reframe/regression.py

@@ -1,4 +1,5 @@
 import reframe as rfm
+import reframe.utility.sanity as sn
 from feelpp.benchmarking.reframe.setup import ReframeSetup, DEBUG
 from feelpp.benchmarking.reframe.config.configReader import FileHandler
 from feelpp.benchmarking.reframe.validation import ValidationHandler
@@ -69,10 +70,15 @@ def copyParametrizedFiles(self):
             self.hashcode
         )
 
-
     @run_before('performance')
     def setPerfVars(self):
         self.perf_variables = {}
+
+        self.perf_variables["__RFM_TOTAL_RUNTIME__"] = sn.make_performance_function(
+            sn.extractsingle( r'__RFM_TOTAL_RUNTIME_SECONDS__=([0-9.]+)', self.stdout, 1, float),
+            unit='s'
+        )
+
         if not self.scalability_handler:
             return
         self.perf_variables.update(
@@ -118,4 +124,4 @@ def sanityCheck(self):
             self.validation_handler.check_success(self.stdout)
             and
             self.validation_handler.check_errors(self.stdout)
-        )
+        )

src/feelpp/benchmarking/reframe/resources.py

@@ -130,6 +130,7 @@ def setResources(resources, rfm_test):
         Returns:
             ReFrameTest: The ReFrame test with the resources configured
         """
+        strategy =  ResourceStrategy()
         if resources.tasks and resources.tasks_per_node:
             strategy = TaskAndTaskPerNodeStrategy()
         elif resources.nodes and resources.tasks_per_node:
@@ -158,4 +159,4 @@ def setResources(resources, rfm_test):
 
         strategy.validate(rfm_test)
 
-        return rfm_test
+        return rfm_test