Skip to content

Commit e3eb809

Browse files
SvenKlaassenSvenKlaassen
committed
update lplr sim
1 parent ef28d95 commit e3eb809

File tree

5 files changed

+51
-77
lines changed

5 files changed

+51
-77
lines changed

‎doc/plm/lplr.qmd‎

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ generate_and_show_styled_table(
6464
display_cols=display_columns_coverage,
6565
n_rep=n_rep_coverage,
6666
level_col="level",
67-
# rename_map={"Learner g": "Learner l"},
6867
coverage_highlight_cols=["Coverage"]
6968
)
7069
```
@@ -78,7 +77,6 @@ generate_and_show_styled_table(
7877
display_cols=display_columns_coverage,
7978
n_rep=n_rep_coverage,
8079
level_col="level",
81-
# rename_map={"Learner g": "Learner l"},
8280
coverage_highlight_cols=["Coverage"]
8381
)
8482
```
@@ -132,16 +130,16 @@ print(metadata_df.T.to_string(header=False))
132130
#| echo: false
133131
134132
# set up data and rename columns
135-
df_coverage = pd.read_csv("../../results/plm/lplr_ate_tune_coverage.csv", index_col=None)
133+
df_coverage_tune = pd.read_csv("../../results/plm/lplr_ate_tune_coverage.csv", index_col=None)
136134
137-
if "repetition" in df_coverage.columns and df_coverage["repetition"].nunique() == 1:
138-
n_rep_coverage = df_coverage["repetition"].unique()[0]
139-
elif "n_rep" in df_coverage.columns and df_coverage["n_rep"].nunique() == 1:
140-
n_rep_coverage = df_coverage["n_rep"].unique()[0]
135+
if "repetition" in df_coverage_tune.columns and df_coverage_tune["repetition"].nunique() == 1:
136+
n_rep_coverage_tune = df_coverage_tune["repetition"].unique()[0]
137+
elif "n_rep" in df_coverage_tune.columns and df_coverage_tune["n_rep"].nunique() == 1:
138+
n_rep_coverage_tune = df_coverage_tune["n_rep"].unique()[0]
141139
else:
142-
n_rep_coverage = "N/A" # Fallback if n_rep cannot be determined
140+
n_rep_coverage_tune = "N/A" # Fallback if n_rep cannot be determined
143141
144-
display_columns_coverage = ["Learner m", "Learner M", "Learner t", "Tuned", "Bias", "CI Length", "Coverage"]
142+
display_columns_coverage_tune = ["Learner m", "Learner M", "Learner t", "Tuned", "Bias", "CI Length", "Coverage", "Loss M", "Loss a", "Loss m"]
145143
```
146144

147145
### Nuisance space
@@ -150,12 +148,11 @@ display_columns_coverage = ["Learner m", "Learner M", "Learner t", "Tuned", "Bia
150148
# | echo: false
151149
152150
generate_and_show_styled_table(
153-
main_df=df_coverage,
151+
main_df=df_coverage_tune,
154152
filters={"level": 0.95, "Score": "nuisance_space"},
155-
display_cols=display_columns_coverage,
156-
n_rep=n_rep_coverage,
153+
display_cols=display_columns_coverage_tune,
154+
n_rep=n_rep_coverage_tune,
157155
level_col="level",
158-
# rename_map={"Learner g": "Learner l"},
159156
coverage_highlight_cols=["Coverage"]
160157
)
161158
```
@@ -164,12 +161,11 @@ generate_and_show_styled_table(
164161
#| echo: false
165162
166163
generate_and_show_styled_table(
167-
main_df=df_coverage,
164+
main_df=df_coverage_tune,
168165
filters={"level": 0.9, "Score": "nuisance_space"},
169-
display_cols=display_columns_coverage,
170-
n_rep=n_rep_coverage,
166+
display_cols=display_columns_coverage_tune,
167+
n_rep=n_rep_coverage_tune,
171168
level_col="level",
172-
# rename_map={"Learner g": "Learner l"},
173169
coverage_highlight_cols=["Coverage"]
174170
)
175171
```
@@ -181,10 +177,10 @@ generate_and_show_styled_table(
181177
#| echo: false
182178
183179
generate_and_show_styled_table(
184-
main_df=df_coverage,
180+
main_df=df_coverage_tune,
185181
filters={"level": 0.95, "Score": "instrument"},
186-
display_cols=display_columns_coverage,
187-
n_rep=n_rep_coverage,
182+
display_cols=display_columns_coverage_tune,
183+
n_rep=n_rep_coverage_tune,
188184
level_col="level",
189185
coverage_highlight_cols=["Coverage"]
190186
)
@@ -194,10 +190,10 @@ generate_and_show_styled_table(
194190
#| echo: false
195191
196192
generate_and_show_styled_table(
197-
main_df=df_coverage,
193+
main_df=df_coverage_tune,
198194
filters={"level": 0.9, "Score": "instrument"},
199-
display_cols=display_columns_coverage,
200-
n_rep=n_rep_coverage,
195+
display_cols=display_columns_coverage_tune,
196+
n_rep=n_rep_coverage_tune,
201197
level_col="level",
202198
coverage_highlight_cols=["Coverage"]
203199
)

‎monte-cover/src/montecover/did/did_pa_multi_tune.py‎

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
from montecover.base import BaseSimulation
1010
from montecover.utils import create_learner_from_config
11+
from montecover.utils_tuning import lgbm_reg_params, lgbm_cls_params
1112

1213

1314
class DIDMultiTuningCoverageSimulation(BaseSimulation):
@@ -34,29 +35,7 @@ def __init__(
3435
self._calculate_oracle_values()
3536

3637
# tuning specific settings
37-
# parameter space for the outcome regression tuning
38-
def ml_g_params(trial):
39-
return {
40-
'n_estimators': 100,
41-
'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.1, log=True),
42-
'max_depth': 3,
43-
'min_child_samples': trial.suggest_int('min_child_samples', 10, 20, step=5),
44-
'lambda_l1': trial.suggest_float('lambda_l1', 1e-2, 10.0, log=True),
45-
'lambda_l2': trial.suggest_float('lambda_l2', 1e-2, 10.0, log=True),
46-
}
47-
48-
# parameter space for the propensity score tuning
49-
def ml_m_params(trial):
50-
return {
51-
'n_estimators': 100,
52-
'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.1, log=True),
53-
'max_depth': 3,
54-
'min_child_samples': trial.suggest_int('min_child_samples', 10, 20, step=5),
55-
'lambda_l1': trial.suggest_float('lambda_l1', 1e-2, 10.0, log=True),
56-
'lambda_l2': trial.suggest_float('lambda_l2', 1e-2, 10.0, log=True),
57-
}
58-
59-
self._param_space = {"ml_g": ml_g_params, "ml_m": ml_m_params}
38+
self._param_space = {"ml_g": lgbm_reg_params, "ml_m": lgbm_cls_params}
6039

6140
self._optuna_settings = {
6241
"n_trials": 50,
@@ -67,7 +46,6 @@ def ml_m_params(trial):
6746
def _process_config_parameters(self):
6847
"""Process simulation-specific parameters from config"""
6948
# Process ML models in parameter grid
70-
# Process ML models in parameter grid
7149
assert (
7250
"learners" in self.dml_parameters
7351
), "No learners specified in the config file"
@@ -156,6 +134,7 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
156134
for model in [dml_model, dml_model_tuned]:
157135
model.fit()
158136
model.bootstrap(n_rep_boot=2000)
137+
nuisance_loss = model.nuisance_loss
159138
for level in self.confidence_parameters["level"]:
160139
level_result = dict()
161140
level_result["detailed"] = self._compute_coverage(
@@ -188,6 +167,9 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
188167
"In-sample-norm.": in_sample_normalization,
189168
"level": level,
190169
"Tuned": model is dml_model_tuned,
170+
"Loss g_control": nuisance_loss["ml_g0"].mean(),
171+
"Loss g_treated": nuisance_loss["ml_g1"].mean(),
172+
"Loss m": nuisance_loss["ml_m"].mean(),
191173
}
192174
)
193175
for key, res in level_result.items():
@@ -214,6 +196,9 @@ def summarize_results(self):
214196
"Bias": "mean",
215197
"Uniform Coverage": "mean",
216198
"Uniform CI Length": "mean",
199+
"Loss g_control": "mean",
200+
"Loss g_treated": "mean",
201+
"Loss m": "mean",
217202
"repetition": "count",
218203
}
219204

‎monte-cover/src/montecover/plm/lplr_ate_tune.py‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
from montecover.base import BaseSimulation
88
from montecover.utils import create_learner_from_config
9+
from montecover.utils_tuning import lgbm_reg_params, lgbm_cls_params
910

1011

1112
class LPLRATETuningCoverageSimulation(BaseSimulation):
@@ -30,26 +31,12 @@ def __init__(
3031
self._calculate_oracle_values()
3132
self._use_failed_scores = use_failed_scores
3233

33-
# for simplicity, we use the same parameter space for all learners
34-
def ml_params(trial):
35-
return {
36-
"n_estimators": trial.suggest_int("n_estimators", 100, 500, step=50),
37-
"learning_rate": trial.suggest_float(
38-
"learning_rate", 1e-3, 0.1, log=True
39-
),
40-
"min_child_samples": trial.suggest_int(
41-
"min_child_samples", 20, 100, step=5
42-
),
43-
"max_depth": trial.suggest_int("max_depth", 3, 10, step=1),
44-
"lambda_l1": trial.suggest_float("lambda_l1", 1e-8, 10.0, log=True),
45-
"lambda_l2": trial.suggest_float("lambda_l2", 1e-8, 10.0, log=True),
46-
}
47-
34+
# tuning specific settings
4835
self._param_space = {
49-
"ml_M": ml_params,
50-
"ml_t": ml_params,
51-
"ml_m": ml_params,
52-
"ml_a": ml_params,
36+
"ml_M": lgbm_cls_params,
37+
"ml_t": lgbm_reg_params,
38+
"ml_m": lgbm_reg_params,
39+
"ml_a": lgbm_reg_params,
5340
}
5441

5542
self._optuna_settings = {
@@ -112,7 +99,7 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
11299
except RuntimeError as e:
113100
self.logger.info(f"Exception during fit: {e}")
114101
return None
115-
102+
nuisance_loss = model.nuisance_loss
116103
for level in self.confidence_parameters["level"]:
117104
level_result = dict()
118105
level_result["coverage"] = self._compute_coverage(
@@ -132,6 +119,9 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
132119
"Score": score,
133120
"level": level,
134121
"Tuned": model is dml_model_tuned,
122+
"Loss M": nuisance_loss["ml_M"].mean(),
123+
"Loss a": nuisance_loss["ml_a"].mean(),
124+
"Loss m": nuisance_loss["ml_m"].mean(),
135125
}
136126
)
137127
for key, res in level_result.items():
@@ -156,6 +146,9 @@ def summarize_results(self):
156146
"Coverage": "mean",
157147
"CI Length": "mean",
158148
"Bias": "mean",
149+
"Loss M": "mean",
150+
"Loss a": "mean",
151+
"Loss m": "mean",
159152
"repetition": "count",
160153
}
161154

‎results/plm/lplr_ate_tune_coverage.csv‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
Learner m,Learner M,Learner t,Score,level,Tuned,Coverage,CI Length,Bias,repetition
2-
LGBM Regr.,LGBM Clas.,LGBM Regr.,instrument,0.9,False,0.91,0.9117258212067718,0.240354871477558,100
3-
LGBM Regr.,LGBM Clas.,LGBM Regr.,instrument,0.9,True,0.95,0.8692681775643711,0.2054770002796413,100
4-
LGBM Regr.,LGBM Clas.,LGBM Regr.,instrument,0.95,False,0.98,1.0863883229855305,0.240354871477558,100
5-
LGBM Regr.,LGBM Clas.,LGBM Regr.,instrument,0.95,True,0.96,1.0357969201737371,0.2054770002796413,100
6-
LGBM Regr.,LGBM Clas.,LGBM Regr.,nuisance_space,0.9,False,0.91,0.7841573908306078,0.18430486050109982,100
7-
LGBM Regr.,LGBM Clas.,LGBM Regr.,nuisance_space,0.9,True,0.86,0.7221800622589235,0.1665060542122647,100
8-
LGBM Regr.,LGBM Clas.,LGBM Regr.,nuisance_space,0.95,False,0.95,0.9343811625885382,0.18430486050109982,100
9-
LGBM Regr.,LGBM Clas.,LGBM Regr.,nuisance_space,0.95,True,0.93,0.8605306205900738,0.1665060542122647,100
1+
Learner m,Learner M,Learner t,Score,level,Tuned,Coverage,CI Length,Bias,Loss M,Loss a,Loss m,repetition
2+
LGBM Regr.,LGBM Clas.,LGBM Regr.,instrument,0.9,False,0.89,0.9503738196199937,0.27958957834627335,0.7313248611019764,0.3524379916385648,0.3761979381237934,100
3+
LGBM Regr.,LGBM Clas.,LGBM Regr.,instrument,0.9,True,0.86,0.7888791069378829,0.2131544006122909,0.6305442473961151,0.33110728634697295,0.335336012157393,100
4+
LGBM Regr.,LGBM Clas.,LGBM Regr.,instrument,0.95,False,0.92,1.1324402535180162,0.27958957834627335,0.7313248611019764,0.3524379916385648,0.3761979381237934,100
5+
LGBM Regr.,LGBM Clas.,LGBM Regr.,instrument,0.95,True,0.92,0.9400074343514756,0.2131544006122909,0.6305442473961151,0.33110728634697295,0.335336012157393,100
6+
LGBM Regr.,LGBM Clas.,LGBM Regr.,nuisance_space,0.9,False,0.89,0.8105540967664803,0.19761923280521046,0.7266071266302078,0.35476756729034437,0.479148010505746,100
7+
LGBM Regr.,LGBM Clas.,LGBM Regr.,nuisance_space,0.9,True,0.87,0.6815753718036465,0.20220182977401963,0.6305431158253273,0.3305870433409719,0.46869360133810006,100
8+
LGBM Regr.,LGBM Clas.,LGBM Regr.,nuisance_space,0.95,False,0.94,0.9658347777291706,0.19761923280521046,0.7266071266302078,0.35476756729034437,0.479148010505746,100
9+
LGBM Regr.,LGBM Clas.,LGBM Regr.,nuisance_space,0.95,True,0.91,0.8121471476829805,0.20220182977401963,0.6305431158253273,0.3305870433409719,0.46869360133810006,100

‎results/plm/lplr_ate_tune_metadata.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File
2-
0.12.dev0,LPLRATETuningCoverageSimulation,2025-11-26 17:47,44.12576818863551,3.12.9,scripts/plm/lplr_ate_tune_config.yml
2+
0.12.dev0,LPLRATETuningCoverageSimulation,2025-12-01 14:39,38.92375212907791,3.12.9,scripts/plm/lplr_ate_tune_config.yml

0 commit comments

Comments
 (0)