diff --git a/examples/sorting/plots.json b/examples/sorting/plots.json index 4fa4cbe99..3c61caa50 100644 --- a/examples/sorting/plots.json +++ b/examples/sorting/plots.json @@ -81,7 +81,8 @@ "transformation": "performance", "xaxis": { "parameter": "elements", "label": "N" }, "yaxis": { "label": "Execution time (s)" }, - "color_axis": { "parameter": "algorithm", "label": "Algorithm" } + "color_axis": { "parameter": "algorithm", "label": "Algorithm" }, + "aggregations":[{"column":"perfvalue","agg":"filter:elapsed"}] } }, { diff --git a/pyproject.toml b/pyproject.toml index cd051b86a..acd32088e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ where = ["src"] 'report/templates/**', 'scripts/data/*', 'scripts/data/website_images/*', + 'scripts/data/supplemental-ui/**/*', 'reframe/templates/**' ] diff --git a/src/feelpp/benchmarking/dashboardRenderer/component/leaf.py b/src/feelpp/benchmarking/dashboardRenderer/component/leaf.py index 18b2f9f47..ac377c181 100644 --- a/src/feelpp/benchmarking/dashboardRenderer/component/leaf.py +++ b/src/feelpp/benchmarking/dashboardRenderer/component/leaf.py @@ -1,3 +1,4 @@ +import tempfile from feelpp.benchmarking.dashboardRenderer.component.base import GraphNode from feelpp.benchmarking.dashboardRenderer.repository.base import Repository from feelpp.benchmarking.dashboardRenderer.views.base import View @@ -87,23 +88,34 @@ def patchTemplateInfo( self, patch : Union[dict,TemplateDataFile], prefix:str, s save (bool): If True, the patch will be written back to the associated data file (e.g., a JSON file) on the filesystem. """ - if save: - template_data_files = [d for d in self.view.template_info.data if isinstance(d,TemplateDataFile) and d.prefix and d.prefix == prefix ] + template_data_files = [d for d in self.view.template_info.data if isinstance(d,TemplateDataFile) and d.prefix and d.prefix == prefix ] - if len( template_data_files ) > 1: - warnings.warn(f"More than one file having prefix {prefix} found. First occurence will be overwritten") + if len( template_data_files ) > 1: + warnings.warn(f"More than one file having prefix {prefix} found. First occurence will be overwritten") - filepath = None - if len( template_data_files ) == 0: - warnings.warn(f"No data files with {prefix} found in {self.id}. Saving this patch will not be possible.") + patch_data = patch.model_dump() if hasattr(patch, "model_dump") else patch + + if not template_data_files: + warnings.warn(f"No data files with {prefix} found in {self.id}. Saving/patching will not be possible.") + else: + target_file = template_data_files[0] + if save: + write_path = os.path.join(self.view.template_data_dir, target_file.filepath) if hasattr(self.view,"template_data_dir") and self.view.template_data_dir else target_file.filepath else: - filepath = template_data_files[0].filepath - format = template_data_files[0].format - - if filepath: - with open( os.path.join( self.view.template_data_dir, filepath ), "w" ) as f: - if format == "json": - json.dump( patch.model_dump(), f ) - else: - f.write( patch ) - self.view.updateTemplateData( {prefix:patch} ) \ No newline at end of file + base_dir = self.view.template_data_dir if hasattr(self.view,"template_data_dir") and self.view.template_data_dir else (os.path.dirname(target_file.filepath) or ".") + + tmp_fd, write_path = tempfile.mkstemp(dir=base_dir, suffix=f".{target_file.format}") + os.close(tmp_fd) + target_file.filepath = write_path + + with open(write_path, "w") as f: + if target_file.format == "json": + json.dump(patch_data, f) + else: + f.write(patch_data) + self.view.updateTemplateData(target_file) + + if not save: #Cleanup temp file + os.remove(write_path) + + self.view.updateTemplateData( {prefix:patch_data} ) diff --git a/src/feelpp/benchmarking/dashboardRenderer/tests/test_node.py b/src/feelpp/benchmarking/dashboardRenderer/tests/test_node.py index 07d9f7523..446fd4a53 100644 --- a/src/feelpp/benchmarking/dashboardRenderer/tests/test_node.py +++ b/src/feelpp/benchmarking/dashboardRenderer/tests/test_node.py @@ -1,6 +1,6 @@ from feelpp.benchmarking.dashboardRenderer.component.base import GraphNode from feelpp.benchmarking.dashboardRenderer.component.leaf import LeafComponent -from feelpp.benchmarking.dashboardRenderer.schemas.dashboardSchema import TemplateDataFile +from feelpp.benchmarking.dashboardRenderer.schemas.dashboardSchema import TemplateDataFile, TemplateInfo import pytest @@ -8,6 +8,7 @@ class MockView: def __init__(self,name = ""): self.name = name self.template_data = {"test":"template_data"} + self.template_info = TemplateInfo(data=[TemplateDataFile(format="json",prefix="meta",filepath="")]) def clone(self): return MockView(f"Cloned {self.name}") def updateTemplateData(self,data): @@ -310,4 +311,4 @@ def test_leafcomponent_patchTemplateInfo_no_save(): leaf.patchTemplateInfo(patch_data, prefix, save=False) assert view.template_data["meta"] == patch_data - # Ensure no file operations occurred (implicitly, as we aren't mocking open/json.dump) \ No newline at end of file + # Ensure no file operations occurred (implicitly, as we aren't mocking open/json.dump) diff --git a/src/feelpp/benchmarking/reframe/__main__.py b/src/feelpp/benchmarking/reframe/__main__.py index 0c97d5010..05a415bca 100644 --- a/src/feelpp/benchmarking/reframe/__main__.py +++ b/src/feelpp/benchmarking/reframe/__main__.py @@ -10,7 +10,12 @@ def main_cli(): parser = Parser() - machine_reader = ConfigReader(parser.args.machine_config,MachineConfig,"machine",dry_run=parser.args.dry_run) + if parser.args.machine_config: + machine_reader = ConfigReader(parser.args.machine_config,MachineConfig,"machine",dry_run=parser.args.dry_run) + else: + machine_reader = ConfigReader(None,MachineConfig,"machine",dry_run=parser.args.dry_run) + machine_reader.config = MachineConfig(machine="default") + #Sets the cachedir and tmpdir directories for containers for platform, dirs in machine_reader.config.containers.items(): @@ -24,7 +29,8 @@ def main_cli(): cmd_builder = CommandBuilder(machine_reader.config,parser) - os.environ["MACHINE_CONFIG_FILEPATH"] = parser.args.machine_config + if parser.args.machine_config: + os.environ["MACHINE_CONFIG_FILEPATH"] = parser.args.machine_config website_config = WebsiteConfigCreator(machine_reader.config.reports_base_dir) @@ -38,7 +44,10 @@ def main_cli(): app_reader = ConfigReader(configs,ConfigFile,"app",dry_run=parser.args.dry_run,additional_readers=[machine_reader]) executable_name = os.path.basename(app_reader.config.executable).split(".")[0] - report_folder_path = cmd_builder.createReportFolder(executable_name,app_reader.config.use_case_name) + if parser.args.dry_run: + report_folder_path = None + else: + report_folder_path = cmd_builder.createReportFolder(executable_name,app_reader.config.use_case_name) #===============PULL IMAGES==================# if not parser.args.dry_run: @@ -73,41 +82,43 @@ def main_cli(): #================================================# #============== UPDATE WEBSITE CONFIG FILE ==============# - common_itempath = (parser.args.move_results or report_folder_path).split("/") - common_itempath = "/".join(common_itempath[:-1 - (common_itempath[-1] == "")]) + if not parser.args.dry_run: + common_itempath = (parser.args.move_results or report_folder_path).split("/") + common_itempath = "/".join(common_itempath[:-1 - (common_itempath[-1] == "")]) - website_config.updateExecutionMapping( - executable_name, machine_reader.config.machine, app_reader.config.use_case_name, - report_itempath = common_itempath - ) + website_config.updateExecutionMapping( + executable_name, machine_reader.config.machine, app_reader.config.use_case_name, + report_itempath = common_itempath + ) - website_config.updateMachine(machine_reader.config.machine) - website_config.updateUseCase(app_reader.config.use_case_name) - website_config.updateApplication(executable_name) + website_config.updateMachine(machine_reader.config.machine) + website_config.updateUseCase(app_reader.config.use_case_name) + website_config.updateApplication(executable_name) - website_config.save() + website_config.save() #======================================================# #============ CREATING RESULT ITEM ================# - with open(os.path.join(report_folder_path,"report.json"),"w") as f: - f.write(json.dumps(app_reader.config.json_report.model_dump())) - - #Copy use case description if existant - FileHandler.copyResource( - app_reader.config.additional_files.description_filepath, - os.path.join(report_folder_path,"partials"), - "description" - ) + if not parser.args.dry_run: + with open(os.path.join(report_folder_path,"report.json"),"w") as f: + f.write(json.dumps(app_reader.config.json_report.model_dump())) + + #Copy use case description if existant + FileHandler.copyResource( + app_reader.config.additional_files.description_filepath, + os.path.join(report_folder_path,"partials"), + "description" + ) #===============================================# try: # ============== LAUNCH REFRAME =======================# - reframe_cmd = cmd_builder.buildCommand( app_reader.config.timeout) + reframe_cmd = cmd_builder.buildCommand( app_reader.config.timeout ) exit_code = subprocess.run(reframe_cmd, shell=True) #======================================================# finally: - if not os.path.exists(os.path.join(report_folder_path,"reframe_report.json")): + if report_folder_path and not os.path.exists(os.path.join(report_folder_path,"reframe_report.json")): if os.path.exists(os.path.join(report_folder_path,"report.json")): os.remove(os.path.join(report_folder_path,"report.json")) os.rmdir(report_folder_path) @@ -116,8 +127,9 @@ def main_cli(): if parser.args.move_results: if not os.path.exists(parser.args.move_results): os.makedirs(parser.args.move_results) - os.rename(os.path.join(report_folder_path,"reframe_report.json"),os.path.join(parser.args.move_results,"reframe_report.json")) - os.rename(os.path.join(report_folder_path,"report.json"),os.path.join(parser.args.move_results,"report.json")) + if report_folder_path: + os.rename(os.path.join(report_folder_path,"reframe_report.json"),os.path.join(parser.args.move_results,"reframe_report.json")) + os.rename(os.path.join(report_folder_path,"report.json"),os.path.join(parser.args.move_results,"report.json")) #======================================================# if parser.args.website: @@ -126,4 +138,4 @@ def main_cli(): subprocess.run(["npm","run","start"]) # return exit_code.returncode - return 0 \ No newline at end of file + return 0 diff --git a/src/feelpp/benchmarking/reframe/commandBuilder.py b/src/feelpp/benchmarking/reframe/commandBuilder.py index c102fdc58..cc2e6df3e 100644 --- a/src/feelpp/benchmarking/reframe/commandBuilder.py +++ b/src/feelpp/benchmarking/reframe/commandBuilder.py @@ -48,7 +48,6 @@ def buildJobOptions(self,timeout): def buildCommand(self,timeout): - assert self.report_folder_path is not None, "Report folder path not set" cmd = [ 'reframe', f'-C {self.buildConfigFilePath()}', @@ -57,7 +56,7 @@ def buildCommand(self,timeout): f'--system={self.machine_config.machine}', f'--exec-policy={self.machine_config.execution_policy}', f'--prefix={self.machine_config.reframe_base_dir}', - f'--report-file={str(os.path.join(self.report_folder_path,"reframe_report.json"))}', + f'--report-file={str(os.path.join(self.report_folder_path,"reframe_report.json"))}' if self.report_folder_path else "", f"{self.buildJobOptions(timeout)}", f'--perflogdir=logs', f'{self.buildExecutionMode()}' diff --git a/src/feelpp/benchmarking/reframe/config/configReader.py b/src/feelpp/benchmarking/reframe/config/configReader.py index 4a884e62d..b21549c47 100644 --- a/src/feelpp/benchmarking/reframe/config/configReader.py +++ b/src/feelpp/benchmarking/reframe/config/configReader.py @@ -118,8 +118,10 @@ def __init__(self, config_paths, schema, name, dry_run=False, additional_readers self.context = { "dry_run":dry_run } if config_paths: self.config = self.load( self.prepareConfigs(config_paths), schema ) + self.original_config = self.config.model_copy() + else: + self.config = None self.name = name - self.original_config = self.config.model_copy() self.processor = TemplateProcessor() for additional_reader in additional_readers: self.updateConfig(TemplateProcessor.flattenDict(additional_reader.config,additional_reader.name)) @@ -169,6 +171,8 @@ def updateConfig(self, flattened_replace = None): flattened_replace: (dict) Containing all key, pair values that indicate the paths to replace. e.g { "replace.this.path": "with_this_value" } If not provided, placeholders will be changed with own confing """ + if not self.config: + return if not flattened_replace: flattened_replace = TemplateProcessor.flattenDict(self.config.model_dump()) self.config = self.schema.model_validate(self.processor.recursiveReplace(self.config.model_dump(),flattened_replace), context=self.context) @@ -180,4 +184,4 @@ def resetConfig(self, additional_readers = []): self.config = self.original_config.model_copy() for additional_reader in additional_readers: self.updateConfig(TemplateProcessor.flattenDict(additional_reader.config,additional_reader.name)) - self.updateConfig() \ No newline at end of file + self.updateConfig() diff --git a/src/feelpp/benchmarking/reframe/config/machineConfigs/gaya/reframe.py b/src/feelpp/benchmarking/reframe/config/machineConfigs/gaya/reframe.py index ec3f077d5..a874b58c3 100644 --- a/src/feelpp/benchmarking/reframe/config/machineConfigs/gaya/reframe.py +++ b/src/feelpp/benchmarking/reframe/config/machineConfigs/gaya/reframe.py @@ -8,7 +8,7 @@ 'partitions': [ { 'name': 'production', - 'scheduler': 'squeue', + 'scheduler': 'slurm', 'launcher': 'srun', 'max_jobs': 8, 'access': ['--partition=production'], @@ -37,7 +37,7 @@ }, { 'name': 'public', - 'scheduler': 'squeue', + 'scheduler': 'slurm', 'launcher': 'srun', 'max_jobs': 8, 'access': ['--partition=public'], @@ -66,7 +66,7 @@ }, { 'name':'gpu', - 'scheduler':'squeue', + 'scheduler':'slurm', 'launcher':'srun', 'max_jobs':4, 'access': ['--partition=gpu'], diff --git a/src/feelpp/benchmarking/reframe/parser.py b/src/feelpp/benchmarking/reframe/parser.py index 2e5c332e6..fabbb9dec 100644 --- a/src/feelpp/benchmarking/reframe/parser.py +++ b/src/feelpp/benchmarking/reframe/parser.py @@ -21,7 +21,7 @@ def processArgs(self): def addArgs(self): """ Add the necessary arguments to the parser""" options = self.parser.add_argument_group("Options") - options.add_argument('--machine-config', '-mc', required=True, type=str, metavar='MACHINE_CONFIG', help='Path to JSON reframe machine configuration file, specific to a system.') + options.add_argument('--machine-config', '-mc', required=False, default=None, type=str, metavar='MACHINE_CONFIG', help='Path to JSON reframe machine configuration file, specific to a system.') options.add_argument('--plots-config', '-pc', required=False, default=None, type=str, help='Path to JSON plots configuration file, used to generate figures. \nIf not provided, no plots will be generated. The plots configuration can also be included in the benchmark configuration file, under the "plots" field.') options.add_argument('--benchmark-config', '-bc', type=str, nargs='+', action='extend', default=[], metavar='CONFIG', help='Paths to JSON benchmark configuration files \nIn combination with --dir, specify only provide basenames for selecting JSON files.') options.add_argument('--custom-rfm-config', '-rc', type=str, required=False, default=None, help="Additional reframe configuration file to use instead of built-in ones. It should correspond the with the --machine-config specifications.") @@ -40,7 +40,8 @@ def addArgs(self): def convertPathsToAbsolute(self): """ Converts arguments that contain paths to absolute. No change is made if absolute paths are provided""" self.args.benchmark_config = [os.path.abspath(c) for c in self.args.benchmark_config] - self.args.machine_config = os.path.abspath(self.args.machine_config) + if self.args.machine_config: + self.args.machine_config = os.path.abspath(self.args.machine_config) if self.args.plots_config: self.args.plots_config = os.path.abspath(self.args.plots_config) @@ -54,10 +55,6 @@ def validate(self): print(f'[Error] --dir and --benchmark-config combination can only handle one DIR') sys.exit(1) - if not self.args.machine_config: - print(f'[Error] --machine-config should be specified') - sys.exit(1) - def checkDirectoriesExist(self): @@ -104,4 +101,4 @@ def listFilesAndExit(self): for config_path in self.args.benchmark_config: print(f"\t> {config_path}") print(f"\nTotal: {len(self.args.benchmark_config)} file(s)") - sys.exit(0) \ No newline at end of file + sys.exit(0) diff --git a/src/feelpp/benchmarking/reframe/regression.py b/src/feelpp/benchmarking/reframe/regression.py index 37bfd515e..33c93b155 100644 --- a/src/feelpp/benchmarking/reframe/regression.py +++ b/src/feelpp/benchmarking/reframe/regression.py @@ -1,4 +1,5 @@ import reframe as rfm +import reframe.utility.sanity as sn from feelpp.benchmarking.reframe.setup import ReframeSetup, DEBUG from feelpp.benchmarking.reframe.config.configReader import FileHandler from feelpp.benchmarking.reframe.validation import ValidationHandler @@ -69,10 +70,15 @@ def copyParametrizedFiles(self): self.hashcode ) - @run_before('performance') def setPerfVars(self): self.perf_variables = {} + + self.perf_variables["__RFM_TOTAL_RUNTIME__"] = sn.make_performance_function( + sn.extractsingle( r'__RFM_TOTAL_RUNTIME_SECONDS__=([0-9.]+)', self.stdout, 1, float), + unit='s' + ) + if not self.scalability_handler: return self.perf_variables.update( @@ -118,4 +124,4 @@ def sanityCheck(self): self.validation_handler.check_success(self.stdout) and self.validation_handler.check_errors(self.stdout) - ) \ No newline at end of file + ) diff --git a/src/feelpp/benchmarking/reframe/resources.py b/src/feelpp/benchmarking/reframe/resources.py index d15cf637a..98dcd5713 100644 --- a/src/feelpp/benchmarking/reframe/resources.py +++ b/src/feelpp/benchmarking/reframe/resources.py @@ -130,6 +130,7 @@ def setResources(resources, rfm_test): Returns: ReFrameTest: The ReFrame test with the resources configured """ + strategy = ResourceStrategy() if resources.tasks and resources.tasks_per_node: strategy = TaskAndTaskPerNodeStrategy() elif resources.nodes and resources.tasks_per_node: @@ -158,4 +159,4 @@ def setResources(resources, rfm_test): strategy.validate(rfm_test) - return rfm_test \ No newline at end of file + return rfm_test diff --git a/src/feelpp/benchmarking/reframe/schemas/benchmarkSchemas.py b/src/feelpp/benchmarking/reframe/schemas/benchmarkSchemas.py index b97a4b756..7615bdf1e 100644 --- a/src/feelpp/benchmarking/reframe/schemas/benchmarkSchemas.py +++ b/src/feelpp/benchmarking/reframe/schemas/benchmarkSchemas.py @@ -20,7 +20,7 @@ class AdditionalFiles(BaseModel): class ConfigFile(BaseModel): executable: str - timeout: Optional[str] = "0-00:05:00" + timeout: Optional[str] = None resources: Optional[Resources] = Resources(tasks=1, exclusive_access=False) platforms:Optional[Dict[str,Platform]] = {"builtin":Platform()} use_case_name: str @@ -52,16 +52,17 @@ def coerce_report(cls, v): @field_validator("timeout",mode="before") @classmethod def validateTimeout(cls,v): - pattern = r'^\d+-\d{1,2}:\d{1,2}:\d{1,2}$' - if not re.match(pattern, v): - raise ValueError(f"Time is not properly formatted (-::) : {v}") - days,time = v.split("-") - hours,minutes,seconds = time.split(":") + if v: + pattern = r'^\d+-\d{1,2}:\d{1,2}:\d{1,2}$' + if not re.match(pattern, v): + raise ValueError(f"Time is not properly formatted (-::) : {v}") + days,time = v.split("-") + hours,minutes,seconds = time.split(":") - assert int(days) >= 0 - assert 24>int(hours)>=0 - assert 60>int(minutes)>=0 - assert 60>int(seconds)>=0 + assert int(days) >= 0 + assert 24>int(hours)>=0 + assert 60>int(minutes)>=0 + assert 60>int(seconds)>=0 return v @@ -98,3 +99,11 @@ def checkPlatforms(cls,v): raise ValueError(f"{k} not implemented") return v + @field_validator("platforms",mode="after") + @classmethod + def addBuiltinPlatform(cls,v): + if "builtin" not in v: + v["builtin"] = Platform() + return v + + diff --git a/src/feelpp/benchmarking/reframe/schemas/defaultJsonReport.py b/src/feelpp/benchmarking/reframe/schemas/defaultJsonReport.py index 799edb2b8..9e603114d 100644 --- a/src/feelpp/benchmarking/reframe/schemas/defaultJsonReport.py +++ b/src/feelpp/benchmarking/reframe/schemas/defaultJsonReport.py @@ -1,7 +1,7 @@ -from pydantic import model_validator, ValidationError, field_validator +from pydantic import Field, model_validator, ValidationError, field_validator from typing import List,Dict, Optional from feelpp.benchmarking.json_report.figures.schemas.plot import Plot, PlotAxis -from feelpp.benchmarking.json_report.schemas.jsonReport import JsonReportSchema +from feelpp.benchmarking.json_report.schemas.jsonReport import JsonReportSchema, SectionNode class DefaultPlotYAxis(PlotAxis): parameter: Optional[str] = "value" @@ -19,6 +19,11 @@ class DefaultPlot(Plot): DEFAULT_DATA = [ { "name":"reframe_json", "filepath":"./reframe_report.json" }, { "type":"DataTable", "name":"reframe_df", "ref":"reframe_json", "preprocessor":"feelpp.benchmarking.report.plugins.reframeReport:runsToDfPreprocessor" }, + { "type":"DataTable", "name":"perfvar_table", "ref":"reframe_df", + "table_options":{ + "group_by":{"columns":["perfvalue"],"agg":"first"} + } + }, { "type":"DataTable", "name":"parameter_table", "ref":"reframe_df", "table_options":{ "computed_columns":{ "logs_link":"f'link:logs/{row[\"testcases.hashcode\"]}.html[Logs]'" }, @@ -31,7 +36,7 @@ class DefaultPlot(Plot): class JsonReportSchemaWithDefaults(JsonReportSchema): - data: List[Dict] = [] + data: List[Dict] = Field(default_factory=lambda: DEFAULT_DATA.copy()) @staticmethod def addToDefault(v:list): @@ -102,3 +107,38 @@ def setDefaultTitle(self): return self + @model_validator(mode="after") + def applyDefaultContent(self): + if not self.contents: + self.contents = [ + SectionNode(**{ + "type":"section", + "title":"Parametrization", + "contents":[ + { + "type":"table", + "ref": "parameter_table", + "layout":{ + "rename":{ + "testcases.time_total":"Total Time (s)", + "testcases.hashcode":"Hash", + "result":"", + "logs_link":"" + }, + "column_order":["result","testcases.hashcode", "testcases.time_total","logs_link"] + }, + "style":{ + "column_align":{ "result":"center" }, + "column_width":{ "result":1,"logs_link":1}, + "classnames":["scrollable","sortable"] + }, + "filter":{ "placeholder":"Filter testcases..." } + }, + { "type":"table","ref":"perfvar_table", "layout":{ + "rename":{"perfvalue":"Performance Variable"}, "column_order":["perfvalue"] + } } + ] + }) + ] + return self + diff --git a/src/feelpp/benchmarking/reframe/schemas/machines.py b/src/feelpp/benchmarking/reframe/schemas/machines.py index c6d0c8538..c2d38e851 100644 --- a/src/feelpp/benchmarking/reframe/schemas/machines.py +++ b/src/feelpp/benchmarking/reframe/schemas/machines.py @@ -28,10 +28,10 @@ class MachineConfig(BaseModel): active: Optional[bool] = True execution_policy:Optional[Literal["serial","async"]] = "serial" reframe_base_dir:Optional[str] = "./reframe/" - reports_base_dir: Optional[str] = "./reports/" + reports_base_dir: Optional[str] = "$PWD/reports/" input_dataset_base_dir:Optional[str] = None input_user_dir:Optional[str] = None - output_app_dir:str + output_app_dir:Optional[str] = None access:Optional[List[str]] = [] env_variables:Optional[Dict] = {} containers:Optional[Dict[str,Container]] = {} @@ -46,15 +46,25 @@ class MachineConfig(BaseModel): @model_validator(mode="after") def parseTargets(self): + targets = None if not self.targets: - if not self.platform or not self.partitions or not self.prog_environments: - raise ValueError("Either specify the `targets` field or the (platform, partitions,prog_environments) fields for a cartesian product.") - return self + if self.platform and self.partitions and self.prog_environments: + targets = [ + f"{p}:{self.platform}:{e}" + for p in self.partitions + for e in self.prog_environments + ] + self.partitions = [] + self.prog_environments = [] + else: + targets = "::" + else: + targets = self.targets - self.targets = self.targets if type(self.targets) == list else [self.targets] + targets = targets if type(targets) == list else [targets] platform = None - for target in self.targets: + for target in targets: split = target.split(":") if len(split) != 3: @@ -116,4 +126,4 @@ def checkInputUserDir(self): return self class ExecutionConfigFile(RootModel): - List[MachineConfig] \ No newline at end of file + List[MachineConfig] diff --git a/src/feelpp/benchmarking/reframe/schemas/resources.py b/src/feelpp/benchmarking/reframe/schemas/resources.py index 5ef03198a..cbd63aad2 100644 --- a/src/feelpp/benchmarking/reframe/schemas/resources.py +++ b/src/feelpp/benchmarking/reframe/schemas/resources.py @@ -16,4 +16,4 @@ def validateResources(self): self.tasks_per_node and self.nodes and not self.tasks or self.tasks and not self.tasks_per_node and not self.nodes ), "Tasks - tasks_per_node - nodes combination is not supported" - return self \ No newline at end of file + return self diff --git a/src/feelpp/benchmarking/reframe/setup.py b/src/feelpp/benchmarking/reframe/setup.py index 6334a6f44..8e04e4a24 100644 --- a/src/feelpp/benchmarking/reframe/setup.py +++ b/src/feelpp/benchmarking/reframe/setup.py @@ -25,11 +25,13 @@ class ReframeSetup(rfm.RunOnlyRegressionTest): #TODO: Find a way to avoid env variables #====================== INIT READERS ==================# - machine_reader = ConfigReader( - str(os.environ.get("MACHINE_CONFIG_FILEPATH")), - MachineConfig, "machine", - "--dry-run" in sys.argv - ) + machine_filepath = os.environ.get("MACHINE_CONFIG_FILEPATH") + if machine_filepath: + machine_reader = ConfigReader( machine_filepath, MachineConfig, "machine", "--dry-run" in sys.argv ) + else: + machine_reader = ConfigReader(None,MachineConfig,"machine",dry_run="--dry-run" in sys.argv) + machine_reader.config = MachineConfig(machine="default") + app_reader = ConfigReader( str(os.environ.get("APP_CONFIG_FILEPATH")), @@ -157,7 +159,6 @@ def checkInputFileDependencies(self): @run_before('run') def setResources(self): ResourceHandler.setResources(self.app_reader.config.resources, self) - self.num_cpus_per_task = 1 @run_before('run') def cleanupDirectories(self): @@ -171,6 +172,16 @@ def setSchedOptions(self): self.job.options += self.machine_reader.config.access self.job.options += ['--threads-per-core=1'] + + @run_before('run') + def wrapCmdInTimer(self): + self.prerun_cmds += ['START_TIME=$(python3 -c "import time; print(time.time())")'] + self.postrun_cmds += [ + 'END_TIME=$(python3 -c "import time; print(time.time())")', + 'RUNTIME=$(python3 -c "print(f\'{float($END_TIME) - float($START_TIME):.9f}\')")', + 'echo "__RFM_TOTAL_RUNTIME_SECONDS__=${RUNTIME}"' + ] + @run_before('run') def setExecutable(self): if self.machine_reader.config.platform == "builtin": diff --git a/src/feelpp/benchmarking/scripts/data/site.yml.j2 b/src/feelpp/benchmarking/scripts/data/site.yml.j2 index 6d9d563a9..1904c50ed 100644 --- a/src/feelpp/benchmarking/scripts/data/site.yml.j2 +++ b/src/feelpp/benchmarking/scripts/data/site.yml.j2 @@ -7,6 +7,7 @@ content: branches: HEAD start_path: docs ui: + supplemental_files: ./docs/antora/supplemental-ui bundle: url: https://github.com/feelpp/antora-ui/releases/latest/download/ui-bundle.zip snapshot: true @@ -15,4 +16,4 @@ output: dir: public asciidoc: extensions: - - '@feelpp/asciidoctor-extensions' \ No newline at end of file + - '@feelpp/asciidoctor-extensions' diff --git a/src/feelpp/benchmarking/scripts/data/supplemental-ui/css/figures.css b/src/feelpp/benchmarking/scripts/data/supplemental-ui/css/figures.css new file mode 100644 index 000000000..efeaf04e7 --- /dev/null +++ b/src/feelpp/benchmarking/scripts/data/supplemental-ui/css/figures.css @@ -0,0 +1,117 @@ +.figure-container { + position: relative; + width: 100%; + max-width: 100%; + box-sizing: border-box; + margin: 1.2rem 0; + background: none; + box-shadow: none; + padding: 0.5rem; + border: 1px solid #ccc; + border-radius: 4px; +} + +.subfigure-container { + position: absolute; + inset: 0; + background: none; + padding: 0; + border: none; +} + +.subfigure-container.active { + position: relative; +} + +.subfigure-container.inactive { + opacity: 0; + pointer-events: none; + z-index: 0; +} + +/* Tabs container: display buttons inline with a bottom border to indicate grouping */ +.tabs-container { + display: flex; + justify-content: flex-start; + gap: 0; + margin-bottom: 0.4rem; +} + + +.figure-tab { + background: none; + border: none; + font-size: 0.9rem; + color: #444; + cursor: pointer; + border: 1px solid #ccc; + padding: 0.2rem 0.5rem; +} + +.figure-tab:hover { + color: #000; /* subtle darkening on hover */ + background: none; /* no hover background */ +} + + +.export-container { + display: flex; + justify-content: flex-end; + gap: 0.3rem; + margin-bottom: 0.3rem; +} + +.export-container button { + background: none; + color: #666; + border: none; + padding: 0; + font-size: 0.75rem; + cursor: pointer; + text-decoration: underline; +} + +.export-container button:hover { + color: #000; +} + +.exampleblock.plot{ + border: 1px solid #ccc; + border-radius: 4px; +} + +/* Example block styling for plots */ +.exampleblock.plot,.exampleblock.grid,.exampleblock.image { + border: none; + padding: 0; + margin: 1.5rem 0; + display: flex; + flex-direction: column; +} + +.exampleblock.plot>.content,.exampleblock.grid>.content ,.exampleblock.image>.content { + order: 1; + padding: 0; + border: none; +} + +.exampleblock.plot>.title,.exampleblock.grid>.title,.exampleblock.image>.title, .plotly-figure-caption{ + order: 2; + text-align: center; + font-style: italic; + margin-top: 0.5rem; +} + +.exampleblock.example { + border-left: 4px solid var(--brand-primary); +} + + +.download-btn.latex-btn { + position: absolute; + top: 12%; + right: 0; + margin: 16px; + border: solid 1px; + padding: 8px; +} \ No newline at end of file diff --git a/src/feelpp/benchmarking/scripts/data/supplemental-ui/css/table.css b/src/feelpp/benchmarking/scripts/data/supplemental-ui/css/table.css new file mode 100644 index 000000000..ffbdf671a --- /dev/null +++ b/src/feelpp/benchmarking/scripts/data/supplemental-ui/css/table.css @@ -0,0 +1,3 @@ +.scrollable { + overflow-x: auto; +} \ No newline at end of file diff --git a/src/feelpp/benchmarking/scripts/data/supplemental-ui/js/figure-helper.js b/src/feelpp/benchmarking/scripts/data/supplemental-ui/js/figure-helper.js new file mode 100644 index 000000000..4d3161843 --- /dev/null +++ b/src/feelpp/benchmarking/scripts/data/supplemental-ui/js/figure-helper.js @@ -0,0 +1,55 @@ +function downloadData(data, filename) { + if (data.length == 1) { + if (data[0].content.length == 1) { + d = data[0].content[0]; + const blob = new Blob([d.data], { type: "plain/text;charset=utf-8" }); + downloadBlob(blob, `${d.title}.${data[0].format}`); + } + else { + downloadFilesAsZip( + [{ format: data[0].format, content: data[0].content }], + filename + ); + } + } + else if (data.length > 1) { + downloadFilesAsZip(data, filename); + } + else { + alert("No data to download"); + } +} + + +function downloadFilesAsZip(data, zipFilename) { + const zip = new JSZip(); + data.forEach((d) => { + d.content.forEach((c) => { + const filename = `${c.title}.${d.format}`; + zip.file(filename, c.data); + }); + }); + zip.generateAsync({ type: "blob" }).then((blob) => downloadBlob(blob, zipFilename)); +} + +function downloadBlob(blob, filename) { + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); +} + +function switchTab(button, tabIndex) { + const figureContainer = button.closest('.figure-container'); + figureContainer.querySelectorAll('.subfigure-container').forEach(subfig => { + subfig.classList.remove('active'); + subfig.classList.add('inactive'); + }); + const activeSubfig = figureContainer.querySelector(`.subfigure-container[data-subfigure='${tabIndex}']`); + activeSubfig.classList.add('active'); + activeSubfig.classList.remove('inactive'); +} diff --git a/src/feelpp/benchmarking/scripts/data/supplemental-ui/js/table-filter-sort.js b/src/feelpp/benchmarking/scripts/data/supplemental-ui/js/table-filter-sort.js new file mode 100644 index 000000000..bc98a9c6f --- /dev/null +++ b/src/feelpp/benchmarking/scripts/data/supplemental-ui/js/table-filter-sort.js @@ -0,0 +1,71 @@ +// table-filter-sort.js +(function () { + function filterTable(table, input) { + if (!table || !input) return; + + input.addEventListener("input", () => { + const term = input.value.toLowerCase(); + const rows = table.querySelectorAll("tbody tr"); + rows.forEach(row => { + row.style.display = row.textContent.toLowerCase().includes(term) ? "" : "none"; + }); + }); + } + + function sortTable(table) { + if (!table) return; + + const headers = table.querySelectorAll("thead th"); + headers.forEach((header, index) => { + header.style.cursor = "pointer"; + + header.addEventListener("click", () => { + const tbody = table.querySelector("tbody"); + const rows = Array.from(tbody.querySelectorAll("tr")); + const asc = header.dataset.sortOrder !== "asc"; + + // reset arrows + headers.forEach(h => { + h.textContent = h.textContent.replace(/[\u25B2\u25BC]/g, "").trim(); + delete h.dataset.sortOrder; + }); + + // set arrow + header.dataset.sortOrder = asc ? "asc" : "desc"; + header.textContent += asc ? " ▲" : " ▼"; + + // sort rows + rows.sort((a, b) => { + const ta = a.children[index].textContent.trim(); + const tb = b.children[index].textContent.trim(); + const na = parseFloat(ta); + const nb = parseFloat(tb); + + if (!isNaN(na) && !isNaN(nb)) return asc ? na - nb : nb - na; + return asc ? ta.localeCompare(tb) : tb.localeCompare(ta); + }); + + rows.forEach(r => tbody.appendChild(r)); + }); + }); + } + + /** + * Initialize a table with sorting & filtering + * @param {HTMLElement} table - the table element + * @param {string} inputId - ID of the input element + */ + function initTable(table, inputId) { + const input = document.getElementById(inputId); + if (!table || !input) { + console.warn("Table or input not found", table, inputId); + return; + } + + filterTable(table, input); + sortTable(table); + } + + // expose globally + window.initTable = initTable; +})(); diff --git a/src/feelpp/benchmarking/scripts/data/supplemental-ui/partials/footer-content.hbs b/src/feelpp/benchmarking/scripts/data/supplemental-ui/partials/footer-content.hbs new file mode 100644 index 000000000..4fc1cfc60 --- /dev/null +++ b/src/feelpp/benchmarking/scripts/data/supplemental-ui/partials/footer-content.hbs @@ -0,0 +1,14 @@ +
+
+
+ + Cemosis logo + +
+ © {{{year}}} Cemosis, Université de Strasbourg +
+
+ + + + diff --git a/src/feelpp/benchmarking/scripts/data/supplemental-ui/partials/header-content.hbs b/src/feelpp/benchmarking/scripts/data/supplemental-ui/partials/header-content.hbs new file mode 100644 index 000000000..9ee582c23 --- /dev/null +++ b/src/feelpp/benchmarking/scripts/data/supplemental-ui/partials/header-content.hbs @@ -0,0 +1,33 @@ + + + +
+ +
diff --git a/src/feelpp/benchmarking/scripts/init.py b/src/feelpp/benchmarking/scripts/init.py index 2213dc6af..489c4a2dc 100644 --- a/src/feelpp/benchmarking/scripts/init.py +++ b/src/feelpp/benchmarking/scripts/init.py @@ -40,6 +40,9 @@ def init(args): for img in glob.glob(os.path.join(script_data_path,"website_images","*")): shutil.copy(img,"docs/modules/ROOT/images") + #Add supplemental UI + shutil.copytree(os.path.join(script_data_path,"supplemental-ui"), "docs/antora/supplemental-ui") + #Create index with open("docs/modules/ROOT/pages/index.adoc","w") as f: f.write(f"= {args.project_title}\n:page-layout: toolboxes\n:page-tags: catalog, catalog-index\n:docdatetime: {datetime.strftime(datetime.now(),'%Y-%m-%dT%H:%M:%S')}\n") @@ -61,11 +64,11 @@ def main_cli(): init_parser = subparsers.add_parser('init') init_parser.add_argument("--destination",'-d', required=False, default=".",type=str,help="Base directory where to initialize antora files.") - init_parser.add_argument("--project-title",'-t', required=True, type=str, help="The title of your project.") - init_parser.add_argument("--project-name",'-n', required=True, type=str, help="The name of your project. Must not contain spaces or any special characters other than underscore (_)") + init_parser.add_argument("--project-title",'-t', required=False, default="My Project", type=str, help="The title of your project.") + init_parser.add_argument("--project-name",'-n', required=False, default="my_project", type=str, help="The name of your project. Must not contain spaces or any special characters other than underscore (_)") init_parser.set_defaults(func=init) args = parser.parse_args() args.func(args) - return 0 \ No newline at end of file + return 0 diff --git a/tests/configtest/test_machine.py b/tests/configtest/test_machine.py index 35e16cc88..ae0c8657d 100644 --- a/tests/configtest/test_machine.py +++ b/tests/configtest/test_machine.py @@ -80,7 +80,7 @@ def test_targetsValidation(self): output_app_dir="path/to/output", targets="partition:apptainer:env" ) - assert config.targets == ["partition:apptainer:env"] + assert config.targets == "partition:apptainer:env" assert config.platform == "apptainer" assert config.partitions == ["partition"] assert config.prog_environments == ["env"] @@ -121,16 +121,6 @@ def test_partitionEnvsProd(self): assert set(config.partitions) == {"partition1", "partition2"} assert set(config.prog_environments) == {"env1", "env2"} - with pytest.raises(ValueError, match="Either specify the `targets` field or the .* fields for a cartesian product"): - MachineConfig( - machine="TestMachine", - reframe_base_dir="path/to/reframe", - reports_base_dir="path/to/reports", - output_app_dir="path/to/output", - platform="apptainer", - partitions=[] - ) - def test_checkContainerTypes(self): """Tests validation of `containers` field.""" valid_containers = { @@ -159,4 +149,4 @@ def test_checkContainerTypes(self): reports_base_dir="path/to/reports", output_app_dir="path/to/output", containers=invalid_containers - ) \ No newline at end of file + ) diff --git a/tests/configtest/test_parser.py b/tests/configtest/test_parser.py index 6daac2b3c..b24e12dc4 100644 --- a/tests/configtest/test_parser.py +++ b/tests/configtest/test_parser.py @@ -38,9 +38,6 @@ def test_validation(self): #two dir and one bench config parser = initParser(['-mc','machine_config.json','-pc','plots_config.json','--dir','tests/configtest','--dir','tests/data','-bc','test_bc.json']) - with pytest.raises(ValueError,match='sys exit called'): - #No machine config - parser = initParser(['-pc','plots_config.json','-bc','test_bc.json']) with pytest.raises(ValueError,match='sys exit called'): #Non-existent dir