rerun irm sim with updated nuisance loss logging

Author: SvenKlaassen

doc/irm/irm.qmd

@@ -251,7 +251,7 @@ generate_and_show_styled_table(
 
 ## Tuning
 
-The simulations are based on the  the [make_irm_data](https://docs.doubleml.org/stable/api/generated/doubleml.irm.datasets.make_irm_data.html)-DGP with $1000$ observations. This is only an example as the untuned version just relies on the default configuration.
+The simulations are based on the  the [make_irm_data](https://docs.doubleml.org/stable/api/generated/doubleml.irm.datasets.make_irm_data.html)-DGP with $500$ observations. This is only an example as the untuned version just relies on the default configuration.
 
 ::: {.callout-note title="Metadata"  collapse="true"}
 
@@ -275,7 +275,7 @@ df_ate_tune_cov = pd.read_csv("../../results/irm/irm_ate_tune_coverage.csv", ind
 assert df_ate_tune_cov["repetition"].nunique() == 1
 n_rep_ate_tune_cov = df_ate_tune_cov["repetition"].unique()[0]
 
-display_columns_ate_tune_cov = ["Learner g", "Learner m", "Tuned", "Bias", "CI Length", "Coverage",]
+display_columns_ate_tune_cov = ["Learner g", "Learner m", "Tuned", "Bias", "CI Length", "Coverage", "Loss g0", "Loss g1", "Loss m"]
 ```
 
 
@@ -286,7 +286,7 @@ generate_and_show_styled_table(
     main_df=df_ate_tune_cov,
     filters={"level": 0.95},
     display_cols=display_columns_ate_tune_cov,
-    n_rep=n_rep_ate_cov,
+    n_rep=n_rep_ate_tune_cov,
     level_col="level",
     coverage_highlight_cols=["Coverage"]
 )
@@ -300,7 +300,7 @@ generate_and_show_styled_table(
     main_df=df_ate_tune_cov,
     filters={"level": 0.9},
     display_cols=display_columns_ate_tune_cov,
-    n_rep=n_rep_ate_cov,
+    n_rep=n_rep_ate_tune_cov,
     level_col="level",
     coverage_highlight_cols=["Coverage"]
 )

monte-cover/src/montecover/irm/irm_ate_tune.py

@@ -37,9 +37,11 @@ def ml_g_params(trial):
                     "learning_rate", 1e-3, 0.1, log=True
                 ),
                 "min_child_samples": trial.suggest_int(
-                    "min_child_samples", 20, 100, step=5
+                    "min_child_samples", 10, 50, step=5
                 ),
-                "max_depth": trial.suggest_int("max_depth", 3, 10, step=1),
+                "max_depth": 3,
+                "feature_fraction": trial.suggest_float("feature_fraction", 0.6, 1),
+                "bagging_fraction": trial.suggest_float("bagging_fraction", 0.6, 1),
                 "lambda_l1": trial.suggest_float("lambda_l1", 1e-8, 10.0, log=True),
                 "lambda_l2": trial.suggest_float("lambda_l2", 1e-8, 10.0, log=True),
             }
@@ -52,9 +54,11 @@ def ml_m_params(trial):
                     "learning_rate", 1e-3, 0.1, log=True
                 ),
                 "min_child_samples": trial.suggest_int(
-                    "min_child_samples", 20, 100, step=5
+                    "min_child_samples", 10, 50, step=5
                 ),
-                "max_depth": trial.suggest_int("max_depth", 3, 10, step=1),
+                "max_depth": 3,
+                "feature_fraction": trial.suggest_float("feature_fraction", 0.6, 1),
+                "bagging_fraction": trial.suggest_float("bagging_fraction", 0.6, 1),
                 "lambda_l1": trial.suggest_float("lambda_l1", 1e-8, 10.0, log=True),
                 "lambda_l2": trial.suggest_float("lambda_l2", 1e-8, 10.0, log=True),
             }
@@ -118,6 +122,7 @@ def run_single_rep(
             "coverage": [],
         }
         for model in [dml_model, dml_model_tuned]:
+            nuisance_loss = model.nuisance_loss
             for level in self.confidence_parameters["level"]:
                 level_result = dict()
                 level_result["coverage"] = self._compute_coverage(
@@ -135,6 +140,9 @@ def run_single_rep(
                             "Learner m": learner_m_name,
                             "level": level,
                             "Tuned": model is dml_model_tuned,
+                            "Loss g0": nuisance_loss["ml_g0"].mean(),
+                            "Loss g1": nuisance_loss["ml_g1"].mean(),
+                            "Loss m": nuisance_loss["ml_m"].mean(),
                         }
                     )
                 for key, res in level_result.items():
@@ -152,6 +160,9 @@ def summarize_results(self):
             "Coverage": "mean",
             "CI Length": "mean",
             "Bias": "mean",
+            "Loss g0": "mean",
+            "Loss g1": "mean",
+            "Loss m": "mean",
             "repetition": "count",
         }
 

results/irm/irm_ate_tune_config.yml

@@ -1,13 +1,13 @@
 simulation_parameters:
-  repetitions: 500
+  repetitions: 200
   max_runtime: 19800
   random_seed: 42
   n_jobs: -2
 dgp_parameters:
   theta:
   - 0.5
   n_obs:
-  - 1000
+  - 500
   dim_x:
   - 5
 learner_definitions:

results/irm/irm_ate_tune_coverage.csv

@@ -1,5 +1,5 @@
-Learner g,Learner m,level,Tuned,Coverage,CI Length,Bias,repetition
-LGBM Regr.,LGBM Clas.,0.9,False,0.952,1.8938962143131024,0.38916017267237296,500
-LGBM Regr.,LGBM Clas.,0.9,True,0.82,0.31269268855505145,0.09112340834220442,500
-LGBM Regr.,LGBM Clas.,0.95,False,0.988,2.256716530692214,0.38916017267237296,500
-LGBM Regr.,LGBM Clas.,0.95,True,0.9,0.3725963196693502,0.09112340834220442,500
+Learner g,Learner m,level,Tuned,Coverage,CI Length,Bias,Loss g0,Loss g1,Loss m,repetition
+LGBM Regr.,LGBM Clas.,0.9,False,0.92,2.643216056843617,0.6446011747627041,1.1132816524651437,1.1357364135690846,0.667476757878396,200
+LGBM Regr.,LGBM Clas.,0.9,True,0.89,0.5145312866755513,0.12860156717199023,0.9989179129452523,1.0613529528468826,0.5188432505665598,200
+LGBM Regr.,LGBM Clas.,0.95,False,0.985,3.1495861941059564,0.6446011747627041,1.1132816524651437,1.1357364135690846,0.667476757878396,200
+LGBM Regr.,LGBM Clas.,0.95,True,0.935,0.6131018433975747,0.12860156717199023,0.9989179129452523,1.0613529528468826,0.5188432505665598,200

results/irm/irm_ate_tune_metadata.csv

@@ -1,2 +1,2 @@
 DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File
-0.12.dev0,IRMATETuningCoverageSimulation,2025-11-24 13:07,97.69014709790548,3.12.9,scripts/irm/irm_ate_tune_config.yml
+0.12.dev0,IRMATETuningCoverageSimulation,2025-12-01 12:02,27.278138709068298,3.12.9,scripts/irm/irm_ate_tune_config.yml

scripts/irm/irm_ate_tune_config.yml

@@ -1,14 +1,14 @@
 # Simulation parameters for IRM ATE Coverage with Tuning
 
 simulation_parameters:
-  repetitions: 500
+  repetitions: 200
   max_runtime: 19800 # 5.5 hours in seconds
   random_seed: 42
   n_jobs: -2
 
 dgp_parameters:
   theta: [0.5] # Treatment effect
-  n_obs: [1000] # Sample size
+  n_obs: [500] # Sample size
   dim_x: [5] # Number of covariates
 
 # Define reusable learner configurations