Skip to content

Commit 84ea55f

Browse files
SvenKlaassenSvenKlaassen
committed
add did
1 parent 10bb9d9 commit 84ea55f

File tree

8 files changed

+50
-74
lines changed

8 files changed

+50
-74
lines changed

‎monte-cover/src/montecover/did/did_pa_multi_tune.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
9191
learner_g_name, ml_g = create_learner_from_config(learner_config["ml_g"])
9292
learner_m_name, ml_m = create_learner_from_config(learner_config["ml_m"])
9393
score = dml_params["score"]
94+
control_group = dml_params["control_group"]
9495
in_sample_normalization = dml_params["in_sample_normalization"]
9596

9697
# Model
@@ -100,6 +101,7 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
100101
ml_m=None if score == "experimental" else ml_m,
101102
gt_combinations="standard",
102103
score=score,
104+
control_group=control_group,
103105
in_sample_normalization=in_sample_normalization,
104106
)
105107
# Tuning
@@ -109,6 +111,7 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
109111
ml_m=None if score == "experimental" else ml_m,
110112
gt_combinations="standard",
111113
score=score,
114+
control_group=control_group,
112115
in_sample_normalization=in_sample_normalization,
113116
)
114117
dml_model_tuned.tune_ml_models(
@@ -164,6 +167,7 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]:
164167
"Learner g": learner_g_name,
165168
"Learner m": learner_m_name,
166169
"Score": score,
170+
"Control Group": control_group,
167171
"In-sample-norm.": in_sample_normalization,
168172
"level": level,
169173
"Tuned": model is dml_model_tuned,
@@ -185,6 +189,7 @@ def summarize_results(self):
185189
"Learner g",
186190
"Learner m",
187191
"Score",
192+
"Control Group",
188193
"In-sample-norm.",
189194
"DGP",
190195
"level",

‎results/did/did_pa_multi_tune_config.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
simulation_parameters:
2-
repetitions: 100
2+
repetitions: 200
33
max_runtime: 19800
44
random_seed: 42
55
n_jobs: -2
66
dgp_parameters:
77
DGP:
88
- 1
9-
- 2
10-
- 3
119
- 4
1210
n_obs:
1311
- 2000
@@ -20,6 +18,8 @@ dml_parameters:
2018
learners:
2119
- ml_g: *id001
2220
ml_m: *id002
21+
control_group:
22+
- never_treated
2323
score:
2424
- observational
2525
in_sample_normalization:

‎results/did/did_pa_multi_tune_detailed.csv‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,9 @@
1-
Learner g,Learner m,Score,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2-
LGBM Regr.,LGBM Clas.,observational,True,1,0.9,False,0.8991666666666667,1.1367782872857282,0.2762362295842985,0.93,1.7739576901791148,3.661560613833878,2.9515587152603526,0.8541707478068831,100
3-
LGBM Regr.,LGBM Clas.,observational,True,1,0.9,True,0.8775,0.6396360988823492,0.1679386739873294,0.83,1.0011531338160695,3.360158161171733,2.7665447739373614,0.6798065307413697,100
4-
LGBM Regr.,LGBM Clas.,observational,True,1,0.95,False,0.9625,1.354554876482567,0.2762362295842985,0.97,1.9460309141153076,3.661560613833878,2.9515587152603526,0.8541707478068831,100
5-
LGBM Regr.,LGBM Clas.,observational,True,1,0.95,True,0.9233333333333333,0.7621734216828837,0.1679386739873294,0.89,1.0961658049458993,3.360158161171733,2.7665447739373614,0.6798065307413697,100
6-
LGBM Regr.,LGBM Clas.,observational,True,2,0.9,False,0.8983333333333333,1.2330356378884049,0.30663270016192556,0.92,1.9173958386413434,3.386477693258062,3.152253900679997,0.8480386768604923,100
7-
LGBM Regr.,LGBM Clas.,observational,True,2,0.9,True,0.8916666666666667,0.6719441811647116,0.16476409918696738,0.86,1.0513367602608807,3.1088298411208632,2.92912769411877,0.6763092050163523,100
8-
LGBM Regr.,LGBM Clas.,observational,True,2,0.95,False,0.9541666666666667,1.4692525841309676,0.30663270016192556,0.96,2.1053464077111452,3.386477693258062,3.152253900679997,0.8480386768604923,100
9-
LGBM Regr.,LGBM Clas.,observational,True,2,0.95,True,0.9425,0.8006708761952025,0.16476409918696738,0.93,1.1526329365873031,3.1088298411208632,2.92912769411877,0.6763092050163523,100
10-
LGBM Regr.,LGBM Clas.,observational,True,3,0.9,False,0.905,1.31358053465385,0.32661174597213943,0.9,2.0440203617896806,2.999802652037201,2.982841611043071,0.8507272462648376,100
11-
LGBM Regr.,LGBM Clas.,observational,True,3,0.9,True,0.865,0.7289212442976212,0.1945603364540163,0.86,1.1371526826681673,3.032548727546919,3.0240798464809977,0.6800265869273724,100
12-
LGBM Regr.,LGBM Clas.,observational,True,3,0.95,False,0.9566666666666667,1.5652277482501924,0.32661174597213943,0.96,2.242653923089107,2.999802652037201,2.982841611043071,0.8507272462648376,100
13-
LGBM Regr.,LGBM Clas.,observational,True,3,0.95,True,0.9308333333333333,0.8685632344303481,0.1945603364540163,0.88,1.2462114970320124,3.032548727546919,3.0240798464809977,0.6800265869273724,100
14-
LGBM Regr.,LGBM Clas.,observational,True,4,0.9,False,0.9258333333333333,1.4012822649889498,0.33623310300936715,0.91,2.1760953165140573,2.9827080885681765,2.9977847813459078,0.8486888442599515,100
15-
LGBM Regr.,LGBM Clas.,observational,True,4,0.9,True,0.7533333333333333,0.775057031885869,0.27368317437679723,0.6,1.2076437086430896,2.996299502921045,3.0408206631075583,0.6757679707228056,100
16-
LGBM Regr.,LGBM Clas.,observational,True,4,0.95,False,0.965,1.6697308055570117,0.33623310300936715,0.94,2.3932554225940295,2.9827080885681765,2.9977847813459078,0.8486888442599515,100
17-
LGBM Regr.,LGBM Clas.,observational,True,4,0.95,True,0.8358333333333333,0.9235374160777122,0.27368317437679723,0.71,1.325967981957511,2.996299502921045,3.0408206631075583,0.6757679707228056,100
1+
Learner g,Learner m,Score,Control Group,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,False,0.9120833333333332,1.1281458608527684,0.27675088039547086,0.94,1.766959072388302,3.597108107523503,2.9403917178509804,0.8534357674606872,200
3+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,True,0.89375,0.6376532313206807,0.15286697800217827,0.915,0.9983993286704501,3.29374008288805,2.748707897836178,0.679591632802887,200
4+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,False,0.9620833333333333,1.3442687059500862,0.27675088039547086,0.97,1.9367883375747112,3.597108107523503,2.9403917178509804,0.8534357674606872,200
5+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,True,0.9541666666666667,0.759810689252895,0.15286697800217827,0.945,1.0938791918349722,3.29374008288805,2.748707897836178,0.679591632802887,200
6+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.9041666666666667,1.4389904260432933,0.3468462138226047,0.91,2.2315687188052546,2.979241703960451,2.9994404795169123,0.8446712058892714,200
7+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.7629166666666667,0.7775230352607071,0.2683054004360395,0.63,1.2089929988521952,2.989557883560226,3.044965814893618,0.6749671427573682,200
8+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,False,0.9554166666666667,1.7146628508033268,0.3468462138226047,0.97,2.451356071426852,2.979241703960451,2.9994404795169123,0.8446712058892714,200
9+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,True,0.8429166666666668,0.926475840337015,0.2683054004360395,0.775,1.3270667190666499,2.989557883560226,3.044965814893618,0.6749671427573682,200

‎results/did/did_pa_multi_tune_eventstudy.csv‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,9 @@
1-
Learner g,Learner m,Score,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2-
LGBM Regr.,LGBM Clas.,observational,True,1,0.9,False,0.9083333333333333,1.0742547796394701,0.2618988136569734,0.94,1.5022639132232276,3.661560613833878,2.9515587152603526,0.8541707478068831,100
3-
LGBM Regr.,LGBM Clas.,observational,True,1,0.9,True,0.8633333333333333,0.5878107921893911,0.15731265501369246,0.88,0.8222540013002478,3.360158161171733,2.7665447739373614,0.6798065307413697,100
4-
LGBM Regr.,LGBM Clas.,observational,True,1,0.95,False,0.9566666666666667,1.2800535219754794,0.2618988136569734,0.98,1.6776898490222303,3.661560613833878,2.9515587152603526,0.8541707478068831,100
5-
LGBM Regr.,LGBM Clas.,observational,True,1,0.95,True,0.9233333333333333,0.7004197598727457,0.15731265501369246,0.91,0.915822574094016,3.360158161171733,2.7665447739373614,0.6798065307413697,100
6-
LGBM Regr.,LGBM Clas.,observational,True,2,0.9,False,0.9016666666666667,1.1885817333189461,0.28095848380634825,0.9,1.6488463476730966,3.386477693258062,3.152253900679997,0.8480386768604923,100
7-
LGBM Regr.,LGBM Clas.,observational,True,2,0.9,True,0.91,0.6233279876475303,0.1499301158544056,0.91,0.8702171833507445,3.1088298411208632,2.92912769411877,0.6763092050163523,100
8-
LGBM Regr.,LGBM Clas.,observational,True,2,0.95,False,0.9533333333333333,1.4162824897099824,0.28095848380634825,0.95,1.844474784460219,3.386477693258062,3.152253900679997,0.8480386768604923,100
9-
LGBM Regr.,LGBM Clas.,observational,True,2,0.95,True,0.9533333333333333,0.7427411085868194,0.1499301158544056,0.94,0.9737952014197065,3.1088298411208632,2.92912769411877,0.6763092050163523,100
10-
LGBM Regr.,LGBM Clas.,observational,True,3,0.9,False,0.91,1.265646440169177,0.3020810902480612,0.96,1.7582545006922015,2.999802652037201,2.982841611043071,0.8507272462648376,100
11-
LGBM Regr.,LGBM Clas.,observational,True,3,0.9,True,0.8516666666666667,0.6864549574097271,0.18550205150287405,0.82,0.9531749334821116,3.032548727546919,3.0240798464809977,0.6800265869273724,100
12-
LGBM Regr.,LGBM Clas.,observational,True,3,0.95,False,0.9566666666666667,1.5081107517697077,0.3020810902480612,0.99,1.9733460594562338,2.999802652037201,2.982841611043071,0.8507272462648376,100
13-
LGBM Regr.,LGBM Clas.,observational,True,3,0.95,True,0.91,0.8179615325563163,0.18550205150287405,0.91,1.0675220384475739,3.032548727546919,3.0240798464809977,0.6800265869273724,100
14-
LGBM Regr.,LGBM Clas.,observational,True,4,0.9,False,0.9233333333333333,1.3639352077948335,0.31325283477445254,0.92,1.8864279971533038,2.9827080885681765,2.9977847813459078,0.8486888442599515,100
15-
LGBM Regr.,LGBM Clas.,observational,True,4,0.9,True,0.675,0.7467959521377197,0.29502185426592087,0.57,1.0327936903874404,2.996299502921045,3.0408206631075583,0.6757679707228056,100
16-
LGBM Regr.,LGBM Clas.,observational,True,4,0.95,False,0.955,1.6252290420993776,0.31325283477445254,0.94,2.1144079580645485,2.9827080885681765,2.9977847813459078,0.8486888442599515,100
17-
LGBM Regr.,LGBM Clas.,observational,True,4,0.95,True,0.76,0.8898622625181539,0.29502185426592087,0.71,1.157187445369494,2.996299502921045,3.0408206631075583,0.6757679707228056,100
1+
Learner g,Learner m,Score,Control Group,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,False,0.9075,1.044931024932891,0.24865990464197016,0.935,1.4671727358061626,3.597108107523503,2.9403917178509804,0.8534357674606872,200
3+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,True,0.8933333333333333,0.5855097565227854,0.13924209096865323,0.925,0.8214814724321132,3.29374008288805,2.748707897836178,0.679591632802887,200
4+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,False,0.9583333333333333,1.245112113101973,0.24865990464197016,0.985,1.6330868182303546,3.597108107523503,2.9403917178509804,0.8534357674606872,200
5+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,True,0.9508333333333333,0.6976779067620538,0.13924209096865323,0.945,0.9148690368152829,3.29374008288805,2.748707897836178,0.679591632802887,200
6+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.92,1.4033187650033796,0.3275689371006418,0.94,1.937780599061532,2.979241703960451,2.9994404795169123,0.8446712058892714,200
7+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.705,0.7446092869675938,0.27966561432340287,0.6,1.0285584537514965,2.989557883560226,3.044965814893618,0.6749671427573682,200
8+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,False,0.9683333333333333,1.6721574449961667,0.3275689371006418,0.975,2.1663251795251472,2.979241703960451,2.9994404795169123,0.8446712058892714,200
9+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,True,0.7883333333333333,0.8872566902596436,0.27966561432340287,0.745,1.153279573853566,2.989557883560226,3.044965814893618,0.6749671427573682,200

‎results/did/did_pa_multi_tune_group.csv‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,9 @@
1-
Learner g,Learner m,Score,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2-
LGBM Regr.,LGBM Clas.,observational,True,1,0.9,False,0.9333333333333332,1.1085497176172174,0.25053137815681464,0.97,1.4092013919200317,3.661560613833878,2.9515587152603526,0.8541707478068831,100
3-
LGBM Regr.,LGBM Clas.,observational,True,1,0.9,True,0.88,0.6270768096547109,0.16475748288364936,0.86,0.7990834496186516,3.360158161171733,2.7665447739373614,0.6798065307413697,100
4-
LGBM Regr.,LGBM Clas.,observational,True,1,0.95,False,0.9766666666666667,1.3209184610722169,0.25053137815681464,0.99,1.594604589934972,3.661560613833878,2.9515587152603526,0.8541707478068831,100
5-
LGBM Regr.,LGBM Clas.,observational,True,1,0.95,True,0.92,0.747208105526932,0.16475748288364936,0.92,0.904463289343679,3.360158161171733,2.7665447739373614,0.6798065307413697,100
6-
LGBM Regr.,LGBM Clas.,observational,True,2,0.9,False,0.91,1.2178833587850213,0.2840522548466761,0.93,1.545309551595824,3.386477693258062,3.152253900679997,0.8480386768604923,100
7-
LGBM Regr.,LGBM Clas.,observational,True,2,0.9,True,0.89,0.6634109544707237,0.1557070964325136,0.95,0.8467228385738873,3.1088298411208632,2.92912769411877,0.6763092050163523,100
8-
LGBM Regr.,LGBM Clas.,observational,True,2,0.95,False,0.9666666666666667,1.4511975299670465,0.2840522548466761,0.98,1.7560977865592113,3.386477693258062,3.152253900679997,0.8480386768604923,100
9-
LGBM Regr.,LGBM Clas.,observational,True,2,0.95,True,0.9566666666666667,0.7905029094423619,0.1557070964325136,0.97,0.9593452409062102,3.1088298411208632,2.92912769411877,0.6763092050163523,100
10-
LGBM Regr.,LGBM Clas.,observational,True,3,0.9,False,0.9266666666666667,1.3054382973147876,0.31275604791627104,0.92,1.6577767591094739,2.999802652037201,2.982841611043071,0.8507272462648376,100
11-
LGBM Regr.,LGBM Clas.,observational,True,3,0.9,True,0.8733333333333333,0.7327385795188934,0.19959731742987738,0.85,0.9301408865604531,3.032548727546919,3.0240798464809977,0.6800265869273724,100
12-
LGBM Regr.,LGBM Clas.,observational,True,3,0.95,False,0.9666666666666667,1.5555256740493915,0.31275604791627104,0.95,1.8751718957945658,2.999802652037201,2.982841611043071,0.8507272462648376,100
13-
LGBM Regr.,LGBM Clas.,observational,True,3,0.95,True,0.9266666666666667,0.8731118699004089,0.19959731742987738,0.92,1.0542511457228387,3.032548727546919,3.0240798464809977,0.6800265869273724,100
14-
LGBM Regr.,LGBM Clas.,observational,True,4,0.9,False,0.9233333333333333,1.3901952872075152,0.3085413587438872,0.94,1.7625009703571424,2.9827080885681765,2.9977847813459078,0.8486888442599515,100
15-
LGBM Regr.,LGBM Clas.,observational,True,4,0.9,True,0.6833333333333332,0.7884798862014211,0.29872708825391375,0.65,1.0051398890297463,2.996299502921045,3.0408206631075583,0.6757679707228056,100
16-
LGBM Regr.,LGBM Clas.,observational,True,4,0.95,False,0.9666666666666667,1.6565198566963024,0.3085413587438872,0.98,1.9995158900115932,2.9827080885681765,2.9977847813459078,0.8486888442599515,100
17-
LGBM Regr.,LGBM Clas.,observational,True,4,0.95,True,0.7733333333333333,0.939531733503372,0.29872708825391375,0.78,1.1340404414042033,2.996299502921045,3.0408206631075583,0.6757679707228056,100
1+
Learner g,Learner m,Score,Control Group,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,False,0.9116666666666667,1.0889543359860974,0.2591017535241754,0.91,1.3866236446708151,3.597108107523503,2.9403917178509804,0.8534357674606872,200
3+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,True,0.89,0.6229752808903669,0.15053905583073077,0.905,0.7933914451788466,3.29374008288805,2.748707897836178,0.679591632802887,200
4+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,False,0.9566666666666667,1.2975691237020013,0.2591017535241754,0.97,1.5696656784049068,3.597108107523503,2.9403917178509804,0.8534357674606872,200
5+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,True,0.9533333333333333,0.7423208325635811,0.15053905583073077,0.96,0.8996786017154156,3.29374008288805,2.748707897836178,0.679591632802887,200
6+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.9133333333333333,1.4454910752551433,0.3253154503600127,0.96,1.8286646948756324,2.979241703960451,2.9994404795169123,0.8446712058892714,200
7+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.74,0.7892476267585741,0.28546431334778133,0.645,1.0046266703621765,2.989557883560226,3.044965814893618,0.6749671427573682,200
8+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,False,0.9616666666666667,1.7224088521025238,0.3253154503600127,0.99,2.07498069683797,2.979241703960451,2.9994404795169123,0.8446712058892714,200
9+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,True,0.825,0.9404465528020834,0.28546431334778133,0.77,1.139020606132637,2.989557883560226,3.044965814893618,0.6749671427573682,200

‎results/did/did_pa_multi_tune_metadata.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File
2-
0.12.dev0,DIDMultiTuningCoverageSimulation,2025-12-01 16:19,97.41892901659011,3.12.9,scripts/did/did_pa_multi_tune_config.yml
2+
0.12.dev0,DIDMultiTuningCoverageSimulation,2025-12-01 21:52,119.05648648738861,3.12.9,scripts/did/did_pa_multi_tune_config.yml

0 commit comments

Comments
 (0)