From 484c3e21fe39d145a7267824b8c9896518ed40c2 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 8 Apr 2026 06:38:19 -0600 Subject: [PATCH 1/4] Test model on simulation data --- dashboard/model_manager.py | 8 ++--- tests/check_model.py | 66 ++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/dashboard/model_manager.py b/dashboard/model_manager.py index d104fbd5..0575d6b6 100644 --- a/dashboard/model_manager.py +++ b/dashboard/model_manager.py @@ -98,11 +98,9 @@ def __init__(self, config_dict, model_type): if model_type not in ("NN", "ensemble_NN", "GP"): raise ValueError(f"Unsupported model type: {model_type}") # Populate inferred calibration in physics units for GUI - # (only meaningful inside the dashboard where state.simulation_calibration is set) - if state.simulation_calibration is not None: - self.populate_inferred_calibration( - config_dict["inputs"], config_dict["outputs"] - ) + self.populate_inferred_calibration( + config_dict["inputs"], config_dict["outputs"] + ) except Exception as e: title = f"Unable to load model {model_type}" msg = f"Error occurred when loading model from MLflow: {e}" diff --git a/tests/check_model.py b/tests/check_model.py index 4e6487b8..a9733d88 100644 --- a/tests/check_model.py +++ b/tests/check_model.py @@ -17,46 +17,38 @@ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "dashboard" ) # similar to "cd ../dashboard" sys.path.insert(0, DASHBOARD_DIR) +from calibration_manager import SimulationCalibrationManager # noqa: E402 from model_manager import ModelManager # noqa: E402 -from utils import load_database, load_data # noqa: E402 +from state_manager import state # noqa: E402 +from utils import load_database, load_data as _load_data # noqa: E402 MODEL_TYPES = ["GP", "NN", "ensemble_NN"] ACCURACY_TOLERANCE = 0.80 -def load_experimental_data(config_dict): - """Fetch all experimental points from the database.""" +def load_data(config_dict): + """Fetch all experimental and simulation points from the database.""" input_names = [v["name"] for v in config_dict["inputs"].values()] output_names = [v["name"] for v in config_dict["outputs"].values()] db = load_database(config_dict) - exp_data, _ = load_data(db, config_dict["experiment"]) + exp_data, sim_data = _load_data(db, config_dict["experiment"]) - return exp_data, input_names, output_names + return exp_data, sim_data, input_names, output_names -def check_evaluate(config_dict, model_type): - """Load model and evaluate with experimental data; verify accuracy (relative RMSE <= threshold).""" - # Load model - mm = ModelManager(config_dict=config_dict, model_type=model_type) - # Load experimental data - df_exp, input_names, output_names = load_experimental_data(config_dict) +def check_accuracy(mm, df, input_names, output_names, label): + """Evaluate model on *df* and return True if all outputs pass the accuracy threshold.""" + if len(df) == 0: + print(f"[SKIP] No {label} data available; skipping accuracy check.") + return True - # Skip accuracy check if no experimental data available - if len(df_exp) == 0: - print( - f"[SKIP] No experimental data available for {config_dict['experiment']}; skipping accuracy check." - ) - return + inputs = {n: torch.tensor(df[n].values) for n in input_names} - # Convert input to the format expected by the model manager - inputs = {n: torch.tensor(df_exp[n].values) for n in input_names} - - # Check accuracy all_passed = True for output_name in output_names: - actual = torch.tensor(df_exp[output_name].values) + actual = torch.tensor(df[output_name].values) if actual.isnan().all(): print( f" [SKIP] Output '{output_name}': all actual values are NaN; skipping." @@ -75,8 +67,34 @@ def check_evaluate(config_dict, model_type): print( f" [{status}] Output '{output_name}': relative RMSE = {rmse:.1%} (tolerance {ACCURACY_TOLERANCE:.0%})" ) + return all_passed + + +def check_evaluate(config_dict, model_type): + """Load model and evaluate with experimental and simulation data; verify accuracy.""" + # Set up calibration so ModelManager can populate inferred values + simulation_calibration = config_dict.get("simulation_calibration", {}) + cal_manager = SimulationCalibrationManager(simulation_calibration) + state.use_inferred_calibration = True + + # Load model (populates inferred calibration in state.simulation_calibration) + mm = ModelManager(config_dict=config_dict, model_type=model_type) + + # Load experimental and simulation data + df_exp, df_sim, input_names, output_names = load_data(config_dict) + + # Check accuracy on experimental data + print("Checking experimental data...") + exp_passed = check_accuracy(mm, df_exp, input_names, output_names, "experimental") + + # Convert simulation data to experimental units using inferred calibration + cal_manager.convert_sim_to_exp(df_sim) + + # Check accuracy on simulation data + print("Checking simulation data...") + sim_passed = check_accuracy(mm, df_sim, input_names, output_names, "simulation") - if not all_passed: + if not (exp_passed and sim_passed): raise RuntimeError( f"Accuracy check failed: relative RMSE exceeded {ACCURACY_TOLERANCE:.0%} for one or more outputs." ) @@ -105,7 +123,7 @@ def check_evaluate(config_dict, model_type): config_dict = yaml.safe_load(f) print(f"Experiment: {config_dict['experiment']}") - # Load model and evaluate with experimental data + # Load model and evaluate with experimental and simulation data try: check_evaluate(config_dict, args.model) except Exception as e: From 793cf807d49ba6b9de24d37afd6d61621bdfd4a7 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 8 Apr 2026 07:26:15 -0600 Subject: [PATCH 2/4] Fix AttributeError when loading ensemble_NN model in ModelManager NNEnsemble does not expose input_transformers/output_transformers directly; those attributes live on each inner TorchModel. Access them via models[0] when the model type is ensemble_NN. Co-Authored-By: Claude Sonnet 4.6 --- dashboard/model_manager.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dashboard/model_manager.py b/dashboard/model_manager.py index d104fbd5..1c1b282c 100644 --- a/dashboard/model_manager.py +++ b/dashboard/model_manager.py @@ -152,7 +152,11 @@ def populate_inferred_calibration(self, input_variables, output_variables): value.pop("beta_inferred", None) # Input calibration - input_transformers = self.__model.input_transformers + # For ensemble_NN, transformers live on each inner TorchModel (not on NNEnsemble itself) + if self.__model_type == "ensemble_NN": + input_transformers = self.__model.models[0].input_transformers + else: + input_transformers = self.__model.input_transformers assert len(input_transformers) == 2, ( f"Expected exactly 2 input transformers (calibration + normalization), " f"but got {len(input_transformers)}." @@ -167,7 +171,11 @@ def populate_inferred_calibration(self, input_variables, output_variables): state.simulation_calibration[key]["beta_inferred"] = float(beta_inferred[i]) # Output calibration - output_transformers = self.__model.output_transformers + # For ensemble_NN, transformers live on each inner TorchModel (not on NNEnsemble itself) + if self.__model_type == "ensemble_NN": + output_transformers = self.__model.models[0].output_transformers + else: + output_transformers = self.__model.output_transformers assert len(output_transformers) == 2, ( f"Expected exactly 2 output transformers (normalization + calibration), " f"but got {len(output_transformers)}." From 9d9df82044039cef61ce87f8c53aca3e98b2f5de Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 9 Apr 2026 08:07:18 -0600 Subject: [PATCH 3/4] Reduce code duplication --- dashboard/calibration_manager.py | 22 ++++++++++++++++++++++ dashboard/model_manager.py | 13 +++---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/dashboard/calibration_manager.py b/dashboard/calibration_manager.py index 219d1320..d395218c 100644 --- a/dashboard/calibration_manager.py +++ b/dashboard/calibration_manager.py @@ -4,6 +4,28 @@ import copy +def populate_inferred_calibration(variables, alpha_values, beta_values): + """ + Write alpha_inferred / beta_inferred into state.simulation_calibration for each + variable that has a matching 'depends_on' entry. + + variables: dict mapping config key -> variable dict with a 'name' field + (i.e. config_dict['inputs'] or config_dict['outputs']) + alpha_values, beta_values: tensors or lists, one value per variable (same order) + """ + depends_on_to_key = { + v["depends_on"]: k + for k, v in state.simulation_calibration.items() + if "depends_on" in v + } + for i, var_info in enumerate(variables.values()): + exp_name = var_info["name"] + if exp_name in depends_on_to_key: + key = depends_on_to_key[exp_name] + state.simulation_calibration[key]["alpha_inferred"] = float(alpha_values[i]) + state.simulation_calibration[key]["beta_inferred"] = float(beta_values[i]) + + class SimulationCalibrationManager: def __init__(self, simulation_calibration): state.simulation_calibration = copy.deepcopy(simulation_calibration) diff --git a/dashboard/model_manager.py b/dashboard/model_manager.py index b9294d36..df00108d 100644 --- a/dashboard/model_manager.py +++ b/dashboard/model_manager.py @@ -10,6 +10,7 @@ from sfapi_client.compute import Machine from trame.widgets import vuetify3 as vuetify from utils import timer, load_config_dict, create_date_filter +from calibration_manager import populate_inferred_calibration from error_manager import add_error from sfapi_manager import monitor_sfapi_job from state_manager import state @@ -162,11 +163,7 @@ def populate_inferred_calibration(self, input_variables, output_variables): input_inferred_calibration = input_transformers[0] alpha_inferred = 1.0 / input_inferred_calibration.coefficient beta_inferred = input_inferred_calibration.offset - for i, key in enumerate(input_variables.keys()): - state.simulation_calibration[key]["alpha_inferred"] = float( - alpha_inferred[i] - ) - state.simulation_calibration[key]["beta_inferred"] = float(beta_inferred[i]) + populate_inferred_calibration(input_variables, alpha_inferred, beta_inferred) # Output calibration # For ensemble_NN, transformers live on each inner TorchModel (not on NNEnsemble itself) @@ -181,11 +178,7 @@ def populate_inferred_calibration(self, input_variables, output_variables): output_inferred_calibration = output_transformers[-1] alpha_inferred = 1.0 / output_inferred_calibration.coefficient beta_inferred = output_inferred_calibration.offset - for i, key in enumerate(output_variables.keys()): - state.simulation_calibration[key]["alpha_inferred"] = float( - alpha_inferred[i] - ) - state.simulation_calibration[key]["beta_inferred"] = float(beta_inferred[i]) + populate_inferred_calibration(output_variables, alpha_inferred, beta_inferred) # Notify Trame that the dict was modified in-place, so the UI updates state.dirty("simulation_calibration") From b9595d837bb18fd35ee7647f2e39026dd2460635 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 9 Apr 2026 08:38:06 -0600 Subject: [PATCH 4/4] Revert "Reduce code duplication" This reverts commit 9d9df82044039cef61ce87f8c53aca3e98b2f5de. --- dashboard/calibration_manager.py | 22 ---------------------- dashboard/model_manager.py | 13 ++++++++++--- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/dashboard/calibration_manager.py b/dashboard/calibration_manager.py index d395218c..219d1320 100644 --- a/dashboard/calibration_manager.py +++ b/dashboard/calibration_manager.py @@ -4,28 +4,6 @@ import copy -def populate_inferred_calibration(variables, alpha_values, beta_values): - """ - Write alpha_inferred / beta_inferred into state.simulation_calibration for each - variable that has a matching 'depends_on' entry. - - variables: dict mapping config key -> variable dict with a 'name' field - (i.e. config_dict['inputs'] or config_dict['outputs']) - alpha_values, beta_values: tensors or lists, one value per variable (same order) - """ - depends_on_to_key = { - v["depends_on"]: k - for k, v in state.simulation_calibration.items() - if "depends_on" in v - } - for i, var_info in enumerate(variables.values()): - exp_name = var_info["name"] - if exp_name in depends_on_to_key: - key = depends_on_to_key[exp_name] - state.simulation_calibration[key]["alpha_inferred"] = float(alpha_values[i]) - state.simulation_calibration[key]["beta_inferred"] = float(beta_values[i]) - - class SimulationCalibrationManager: def __init__(self, simulation_calibration): state.simulation_calibration = copy.deepcopy(simulation_calibration) diff --git a/dashboard/model_manager.py b/dashboard/model_manager.py index df00108d..b9294d36 100644 --- a/dashboard/model_manager.py +++ b/dashboard/model_manager.py @@ -10,7 +10,6 @@ from sfapi_client.compute import Machine from trame.widgets import vuetify3 as vuetify from utils import timer, load_config_dict, create_date_filter -from calibration_manager import populate_inferred_calibration from error_manager import add_error from sfapi_manager import monitor_sfapi_job from state_manager import state @@ -163,7 +162,11 @@ def populate_inferred_calibration(self, input_variables, output_variables): input_inferred_calibration = input_transformers[0] alpha_inferred = 1.0 / input_inferred_calibration.coefficient beta_inferred = input_inferred_calibration.offset - populate_inferred_calibration(input_variables, alpha_inferred, beta_inferred) + for i, key in enumerate(input_variables.keys()): + state.simulation_calibration[key]["alpha_inferred"] = float( + alpha_inferred[i] + ) + state.simulation_calibration[key]["beta_inferred"] = float(beta_inferred[i]) # Output calibration # For ensemble_NN, transformers live on each inner TorchModel (not on NNEnsemble itself) @@ -178,7 +181,11 @@ def populate_inferred_calibration(self, input_variables, output_variables): output_inferred_calibration = output_transformers[-1] alpha_inferred = 1.0 / output_inferred_calibration.coefficient beta_inferred = output_inferred_calibration.offset - populate_inferred_calibration(output_variables, alpha_inferred, beta_inferred) + for i, key in enumerate(output_variables.keys()): + state.simulation_calibration[key]["alpha_inferred"] = float( + alpha_inferred[i] + ) + state.simulation_calibration[key]["beta_inferred"] = float(beta_inferred[i]) # Notify Trame that the dict was modified in-place, so the UI updates state.dirty("simulation_calibration")