update APO models documentation and enhance APOSTuningCoverageSimulation with loss metrics and parameter tuning

Author: SvenKlaassen

doc/irm/apo.qmd

@@ -84,7 +84,7 @@ generate_and_show_styled_table(
 
 The simulations are based on the  the [make_irm_data_discrete_treatments](https://docs.doubleml.org/stable/api/datasets.html#dataset-generators)-DGP with $500$ observations. Due to the linearity of the DGP, Lasso and Logit Regression are nearly optimal choices for the nuisance estimation.
 
-The non-uniform results (coverage, ci length and bias) refer to averaged values over all quantiles (point-wise confidende intervals).
+The non-uniform results (coverage, ci length and bias) refer to averaged values over all levels (point-wise confidende intervals).
 
 ::: {.callout-note title="Metadata"  collapse="true"}
 
@@ -140,7 +140,7 @@ generate_and_show_styled_table(
 
 The simulations are based on the  the [make_irm_data_discrete_treatments](https://docs.doubleml.org/stable/api/datasets.html#dataset-generators)-DGP with $500$ observations. Due to the linearity of the DGP, Lasso and Logit Regression are nearly optimal choices for the nuisance estimation.
 
-The non-uniform results (coverage, ci length and bias) refer to averaged values over all quantiles (point-wise confidende intervals).
+The non-uniform results (coverage, ci length and bias) refer to averaged values over all levels (point-wise confidende intervals).
 
 ::: {.callout-note title="Metadata"  collapse="true"}
 
@@ -199,7 +199,7 @@ The simulations are based on the  the [make_irm_data_discrete_treatments](https:
 
 ### APOS Coverage
 
-The non-uniform results (coverage, ci length and bias) refer to averaged values over all quantiles (point-wise confidende intervals).
+The non-uniform results (coverage, ci length and bias) refer to averaged values over all levels (point-wise confidende intervals). The same holds for the loss values which are averaged over all treatment levels.
 
 ::: {.callout-note title="Metadata"  collapse="true"}
 
@@ -216,22 +216,22 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_apos = pd.read_csv("../../results/irm/apos_tune_coverage.csv", index_col=None)
+df_apos_tune = pd.read_csv("../../results/irm/apos_tune_coverage.csv", index_col=None)
 
-assert df_apos["repetition"].nunique() == 1
-n_rep_apos = df_apos["repetition"].unique()[0]
+assert df_apos_tune["repetition"].nunique() == 1
+n_rep_apos_tune = df_apos_tune["repetition"].unique()[0]
 
-display_columns_apos = ["Learner g", "Learner m", "Tuned", "Bias", "CI Length", "Coverage", "Uniform CI Length", "Uniform Coverage"]
+display_columns_apos_tune = ["Learner g", "Learner m", "Tuned", "Bias", "CI Length", "Coverage", "Uniform CI Length", "Uniform Coverage", "Loss g_control", "Loss g_treated", "Loss m"]
 ```
 
 ```{python}
 #| echo: false
 
 generate_and_show_styled_table(
-    main_df=df_apos,
+    main_df=df_apos_tune,
     filters={"level": 0.95},
-    display_cols=display_columns_apos,
-    n_rep=n_rep_apos,
+    display_cols=display_columns_apos_tune,
+    n_rep=n_rep_apos_tune,
     level_col="level",
     coverage_highlight_cols=["Coverage", "Uniform Coverage"]
 )
@@ -242,10 +242,10 @@ generate_and_show_styled_table(
 #| echo: false
 
 generate_and_show_styled_table(
-    main_df=df_apos,
+    main_df=df_apos_tune,
     filters={"level": 0.9},
-    display_cols=display_columns_apos,
-    n_rep=n_rep_apos,
+    display_cols=display_columns_apos_tune,
+    n_rep=n_rep_apos_tune,
     level_col="level",
     coverage_highlight_cols=["Coverage", "Uniform Coverage"]
 )
@@ -254,7 +254,7 @@ generate_and_show_styled_table(
 
 ### Causal Contrast Coverage
 
-The non-uniform results (coverage, ci length and bias) refer to averaged values over all quantiles (point-wise confidende intervals).
+The non-uniform results (coverage, ci length and bias) refer to averaged values over all quantiles (point-wise confidende intervals). The same holds for the loss values which are averaged over all treatment levels.
 
 
 ::: {.callout-note title="Metadata"  collapse="true"}
@@ -272,22 +272,22 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_contrast = pd.read_csv("../../results/irm/apos_tune_causal_contrast.csv", index_col=None)
+df_contrast_tune = pd.read_csv("../../results/irm/apos_tune_causal_contrast.csv", index_col=None)
 
-assert df_contrast["repetition"].nunique() == 1
-n_rep_contrast = df_contrast["repetition"].unique()[0]
+assert df_contrast_tune["repetition"].nunique() == 1
+n_rep_contrast_tune = df_contrast_tune["repetition"].unique()[0]
 
-display_columns_contrast = ["Learner g", "Learner m", "Tuned", "Bias", "CI Length", "Coverage", "Uniform CI Length", "Uniform Coverage"]
+display_columns_contrast_tune = ["Learner g", "Learner m", "Tuned", "Bias", "CI Length", "Coverage", "Uniform CI Length", "Uniform Coverage", "Loss g_control", "Loss g_treated", "Loss m"]
 ```
 
 ```{python}
 #| echo: false
 
 generate_and_show_styled_table(
-    main_df=df_contrast,
+    main_df=df_contrast_tune,
     filters={"level": 0.95},
-    display_cols=display_columns_contrast,
-    n_rep=n_rep_contrast,
+    display_cols=display_columns_contrast_tune,
+    n_rep=n_rep_contrast_tune,
     level_col="level",
     coverage_highlight_cols=["Coverage", "Uniform Coverage"]
 )
@@ -298,10 +298,10 @@ generate_and_show_styled_table(
 #| echo: false
 
 generate_and_show_styled_table(
-    main_df=df_contrast,
+    main_df=df_contrast_tune,
     filters={"level": 0.9},
-    display_cols=display_columns_contrast,
-    n_rep=n_rep_contrast,
+    display_cols=display_columns_contrast_tune,
+    n_rep=n_rep_contrast_tune,
     level_col="level",
     coverage_highlight_cols=["Coverage", "Uniform Coverage"]
 )

monte-cover/src/montecover/irm/apos_tune.py

@@ -8,6 +8,7 @@
 
 from montecover.base import BaseSimulation
 from montecover.utils import create_learner_from_config
+from montecover.utils_tuning import lgbm_reg_params, lgbm_cls_params
 
 
 class APOSTuningCoverageSimulation(BaseSimulation):
@@ -31,37 +32,7 @@ def __init__(
         self._calculate_oracle_values()
 
         # tuning specific settings
-        # parameter space for the outcome regression tuning
-        def ml_g_params(trial):
-            return {
-                "n_estimators": trial.suggest_int("n_estimators", 100, 200, step=50),
-                "learning_rate": trial.suggest_float(
-                    "learning_rate", 1e-3, 0.1, log=True
-                ),
-                "min_child_samples": trial.suggest_int(
-                    "min_child_samples", 20, 50, step=5
-                ),
-                "max_depth": 5,
-                "lambda_l1": trial.suggest_float("lambda_l1", 1e-3, 10.0, log=True),
-                "lambda_l2": trial.suggest_float("lambda_l2", 1e-3, 10.0, log=True),
-            }
-
-        # parameter space for the propensity score tuning
-        def ml_m_params(trial):
-            return {
-                "n_estimators": trial.suggest_int("n_estimators", 100, 200, step=50),
-                "learning_rate": trial.suggest_float(
-                    "learning_rate", 1e-3, 0.1, log=True
-                ),
-                "min_child_samples": trial.suggest_int(
-                    "min_child_samples", 20, 50, step=5
-                ),
-                "max_depth": 5,
-                "lambda_l1": trial.suggest_float("lambda_l1", 1e-3, 10.0, log=True),
-                "lambda_l2": trial.suggest_float("lambda_l2", 1e-3, 10.0, log=True),
-            }
-
-        self._param_space = {"ml_g": ml_g_params, "ml_m": ml_m_params}
+        self._param_space = {"ml_g": lgbm_reg_params, "ml_m": lgbm_cls_params}
 
         self._optuna_settings = {
             "n_trials": 200,
@@ -155,6 +126,18 @@ def run_single_rep(
             model.bootstrap(n_rep_boot=2000)
             causal_contrast_model = model.causal_contrast(reference_levels=0)
             causal_contrast_model.bootstrap(n_rep_boot=2000)
+
+            # average all nuisance losses over treatment levels
+            n_lvls = len(model.modellist)
+            loss_dict = {
+                "ml_g_d_lvl0": np.full(n_lvls, np.nan),
+                "ml_g_d_lvl1": np.full(n_lvls, np.nan),
+                "ml_m": np.full(n_lvls, np.nan)
+            }
+            for key in loss_dict.keys():
+                for i_submodel, submodel in enumerate(model.modellist):
+                    loss_dict[key][i_submodel] = submodel.nuisance_loss[key].mean()
+
             for level in self.confidence_parameters["level"]:
                 level_result = dict()
                 level_result["coverage"] = self._compute_coverage(
@@ -180,6 +163,9 @@ def run_single_rep(
                             "Learner m": learner_m_name,
                             "level": level,
                             "Tuned": model is dml_model_tuned,
+                            "Loss g_control": loss_dict["ml_g_d_lvl0"].mean(),
+                            "Loss g_treated": loss_dict["ml_g_d_lvl1"].mean(),
+                            "Loss m": loss_dict["ml_m"].mean(),
                         }
                     )
                 for key, res in level_result.items():
@@ -199,6 +185,9 @@ def summarize_results(self):
             "Bias": "mean",
             "Uniform Coverage": "mean",
             "Uniform CI Length": "mean",
+            "Loss g_control": "mean",
+            "Loss g_treated": "mean",
+            "Loss m": "mean",
             "repetition": "count",
         }
 

results/irm/apos_tune_causal_contrast.csv

@@ -1,5 +1,5 @@
-Learner g,Learner m,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition
-LGBM Regr.,LGBM Clas.,0.9,False,0.915,37.253979808129365,8.903634313073434,0.95,44.16923532073818,200
-LGBM Regr.,LGBM Clas.,0.9,True,0.835,4.834836017081165,1.3591722777708926,0.815,5.703081839207788,200
-LGBM Regr.,LGBM Clas.,0.95,False,0.98,44.39085491153563,8.903634313073434,0.985,50.67060073707776,200
-LGBM Regr.,LGBM Clas.,0.95,True,0.915,5.761062449185174,1.3591722777708926,0.895,6.546100271377481,200
+Learner g,Learner m,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
+LGBM Regr.,LGBM Clas.,0.9,False,0.905,37.557614215103456,9.702752376788483,0.93,44.43580521211171,10.231838281825558,13.632270699354638,0.7977017509425842,200
+LGBM Regr.,LGBM Clas.,0.9,True,0.8625,4.281908161317747,1.1456270552515193,0.885,5.058837856166666,9.74905255354395,11.553230793169227,0.6041491925910187,200
+LGBM Regr.,LGBM Clas.,0.95,False,0.9625,44.75265762296555,9.702752376788483,0.975,51.03212790722511,10.231838281825558,13.632270699354638,0.7977017509425842,200
+LGBM Regr.,LGBM Clas.,0.95,True,0.945,5.102208271774997,1.1456270552515193,0.95,5.809070370902955,9.74905255354395,11.553230793169227,0.6041491925910187,200

results/irm/apos_tune_coverage.csv

@@ -1,5 +1,5 @@
-Learner g,Learner m,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition
-LGBM Regr.,LGBM Clas.,0.9,False,0.93,27.82005877126217,6.439524063849977,0.96,35.676118532942496,200
-LGBM Regr.,LGBM Clas.,0.9,True,0.885,6.300962875030208,1.5916287837149021,0.88,7.710244822536492,200
-LGBM Regr.,LGBM Clas.,0.95,False,0.9766666666666667,33.14964465289176,6.439524063849977,0.975,40.33245838311548,200
-LGBM Regr.,LGBM Clas.,0.95,True,0.9466666666666668,7.50806035298818,1.5916287837149021,0.94,8.829514768318946,200
+Learner g,Learner m,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
+LGBM Regr.,LGBM Clas.,0.9,False,0.9133333333333333,28.052950188600192,7.055413388225961,0.945,35.926212976449165,10.231838281825558,13.632270699354638,0.7977017509425842,200
+LGBM Regr.,LGBM Clas.,0.9,True,0.8866666666666667,6.138357605002483,1.524949927232772,0.865,7.417779175659066,9.74905255354395,11.553230793169227,0.6041491925910187,200
+LGBM Regr.,LGBM Clas.,0.95,False,0.9766666666666667,33.42715189293536,7.055413388225961,0.98,40.556851028772705,10.231838281825558,13.632270699354638,0.7977017509425842,200
+LGBM Regr.,LGBM Clas.,0.95,True,0.945,7.31430422312426,1.524949927232772,0.94,8.537534779534262,9.74905255354395,11.553230793169227,0.6041491925910187,200

results/irm/apos_tune_metadata.csv

@@ -1,2 +1,2 @@
 DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File
-0.12.dev0,APOSTuningCoverageSimulation,2025-11-26 11:38,31.90539586544037,3.12.9,scripts/irm/apos_tune_config.yml
+0.12.dev0,APOSTuningCoverageSimulation,2025-12-01 13:09,38.63118334611257,3.12.9,scripts/irm/apos_tune_config.yml