refactor: update PLR ATE tuning coverage simulation with loss metrics and reduce repetitions

Author: SvenKlaassen

doc/plm/plr.qmd

@@ -233,7 +233,7 @@ df_tune_cov = pd.read_csv("../../results/plm/plr_ate_tune_coverage.csv", index_c
 assert df_tune_cov["repetition"].nunique() == 1
 n_rep_tune_cov = df_tune_cov["repetition"].unique()[0]
 
-display_columns_tune_cov = ["Learner g", "Learner m", "Tuned", "Bias", "CI Length", "Coverage",]
+display_columns_tune_cov = ["Learner g", "Learner m", "Tuned", "Bias", "CI Length", "Coverage", "Loss g", "Loss m"]
 ```
 
 
@@ -248,7 +248,7 @@ generate_and_show_styled_table(
     display_cols=display_columns_tune_cov,
     n_rep=n_rep_tune_cov,
     level_col="level",
-    rename_map={"Learner g": "Learner l"},
+    rename_map={"Learner g": "Learner l", "Loss g": "Loss l"},
     coverage_highlight_cols=["Coverage"]
 )
 ```
@@ -262,7 +262,7 @@ generate_and_show_styled_table(
     display_cols=display_columns_tune_cov,
     n_rep=n_rep_tune_cov,
     level_col="level",
-    rename_map={"Learner g": "Learner l"},
+    rename_map={"Learner g": "Learner l", "Loss g": "Loss l"},
     coverage_highlight_cols=["Coverage"]
 )
 ```

monte-cover/src/montecover/plm/plr_ate_tune.py

@@ -6,6 +6,7 @@
 
 from montecover.base import BaseSimulation
 from montecover.utils import create_learner_from_config
+from montecover.utils_tuning import lgbm_reg_params
 
 
 class PLRATETuningCoverageSimulation(BaseSimulation):
@@ -29,40 +30,10 @@ def __init__(
         self._calculate_oracle_values()
 
         # tuning specific settings
-        # parameter space for the outcome regression tuning
-        def ml_l_params(trial):
-            return {
-                "n_estimators": trial.suggest_int("n_estimators", 100, 500, step=50),
-                "learning_rate": trial.suggest_float(
-                    "learning_rate", 1e-3, 0.1, log=True
-                ),
-                "min_child_samples": trial.suggest_int(
-                    "min_child_samples", 20, 100, step=5
-                ),
-                "max_depth": trial.suggest_int("max_depth", 3, 10, step=1),
-                "lambda_l1": trial.suggest_float("lambda_l1", 1e-8, 10.0, log=True),
-                "lambda_l2": trial.suggest_float("lambda_l2", 1e-8, 10.0, log=True),
-            }
-
-        # parameter space for the propensity score tuning
-        def ml_m_params(trial):
-            return {
-                "n_estimators": trial.suggest_int("n_estimators", 100, 500, step=50),
-                "learning_rate": trial.suggest_float(
-                    "learning_rate", 1e-3, 0.1, log=True
-                ),
-                "min_child_samples": trial.suggest_int(
-                    "min_child_samples", 20, 100, step=5
-                ),
-                "max_depth": trial.suggest_int("max_depth", 3, 10, step=1),
-                "lambda_l1": trial.suggest_float("lambda_l1", 1e-8, 10.0, log=True),
-                "lambda_l2": trial.suggest_float("lambda_l2", 1e-8, 10.0, log=True),
-            }
-
-        self._param_space = {"ml_l": ml_l_params, "ml_m": ml_m_params}
+        self._param_space = {"ml_l": lgbm_reg_params, "ml_m": lgbm_reg_params}
 
         self._optuna_settings = {
-            "n_trials": 500,
+            "n_trials": 200,
             "show_progress_bar": False,
             "verbosity": optuna.logging.WARNING,  # Suppress Optuna logs
         }
@@ -121,6 +92,7 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
             "coverage": [],
         }
         for model in [dml_model, dml_model_tuned]:
+            nuisance_loss = model.nuisance_loss
             for level in self.confidence_parameters["level"]:
                 level_result = dict()
                 level_result["coverage"] = self._compute_coverage(
@@ -139,6 +111,8 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
                             "Score": score,
                             "level": level,
                             "Tuned": model is dml_model_tuned,
+                            "Loss g": nuisance_loss["ml_l"].mean() if score == "partialling out" else nuisance_loss["ml_g"].mean(),
+                            "Loss m": nuisance_loss["ml_m"].mean(),
                         }
                     )
                 for key, res in level_result.items():
@@ -156,6 +130,8 @@ def summarize_results(self):
             "Coverage": "mean",
             "CI Length": "mean",
             "Bias": "mean",
+            "Loss g": "mean",
+            "Loss m": "mean",
             "repetition": "count",
         }
 

results/plm/plr_ate_tune_config.yml

@@ -1,5 +1,5 @@
 simulation_parameters:
-  repetitions: 500
+  repetitions: 200
   max_runtime: 19800
   random_seed: 42
   n_jobs: -2

results/plm/plr_ate_tune_coverage.csv

@@ -1,5 +1,5 @@
-Learner g,Learner m,Score,level,Tuned,Coverage,CI Length,Bias,repetition
-LGBM Regr.,LGBM Regr.,partialling out,0.9,False,0.81,0.1479047026981892,0.04545068026009308,500
-LGBM Regr.,LGBM Regr.,partialling out,0.9,True,0.894,0.1451364370890545,0.037424408565466506,500
-LGBM Regr.,LGBM Regr.,partialling out,0.95,False,0.898,0.17623932347696217,0.04545068026009308,500
-LGBM Regr.,LGBM Regr.,partialling out,0.95,True,0.942,0.1729407315508218,0.037424408565466506,500
+Learner g,Learner m,Score,level,Tuned,Coverage,CI Length,Bias,Loss g,Loss m,repetition
+LGBM Regr.,LGBM Regr.,partialling out,0.9,False,0.805,0.14733663739723357,0.04709716445165978,1.2426113404810248,1.1115356809360686,200
+LGBM Regr.,LGBM Regr.,partialling out,0.9,True,0.845,0.14434980329449199,0.03915064046361367,1.1704317749756923,1.06174632188426,200
+LGBM Regr.,LGBM Regr.,partialling out,0.95,False,0.87,0.17556243192108348,0.04709716445165978,1.2426113404810248,1.1115356809360686,200
+LGBM Regr.,LGBM Regr.,partialling out,0.95,True,0.895,0.17200339957118416,0.03915064046361367,1.1704317749756923,1.06174632188426,200

results/plm/plr_ate_tune_metadata.csv

@@ -1,2 +1,2 @@
 DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File
-0.12.dev0,PLRATETuningCoverageSimulation,2025-11-24 15:37,119.77829658587774,3.12.9,scripts/plm/plr_ate_tune_config.yml
+0.12.dev0,PLRATETuningCoverageSimulation,2025-12-01 13:43,18.97071567773819,3.12.9,scripts/plm/plr_ate_tune_config.yml

scripts/plm/plr_ate_tune_config.yml

@@ -1,7 +1,7 @@
 # Simulation parameters for PLR ATE Coverage
 
 simulation_parameters:
-  repetitions: 500
+  repetitions: 200
   max_runtime: 19800 # 5.5 hours in seconds
   random_seed: 42
   n_jobs: -2