Skip to content

Commit 2cb8aae

Browse files
SvenKlaassenSvenKlaassen
committed
refactor: update DGP specifications and sample size in configuration files; adjust simulation results accordingly
1 parent 84ea55f commit 2cb8aae

File tree

8 files changed

+35
-41
lines changed

8 files changed

+35
-41
lines changed

‎monte-cover/src/montecover/did/did_pa_multi_tune.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ def __init__(
3333

3434
# Calculate oracle values
3535
self._calculate_oracle_values()
36-
3736
# tuning specific settings
3837
self._param_space = {"ml_g": lgbm_reg_params, "ml_m": lgbm_cls_params}
3938

@@ -218,7 +217,7 @@ def summarize_results(self):
218217

219218
def _generate_dml_data(self, dgp_params) -> dml.data.DoubleMLPanelData:
220219
"""Generate data for the simulation."""
221-
data = make_did_CS2021(n_obs=dgp_params["n_obs"], dgp_type=dgp_params["DGP"])
220+
data = make_did_CS2021(n_obs=dgp_params["n_obs"], dgp_type=dgp_params["DGP"], xi=dgp_params.get("xi", 0.0))
222221
dml_data = dml.data.DoubleMLPanelData(
223222
data,
224223
y_col="y",

‎results/did/did_pa_multi_tune_config.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,13 @@ simulation_parameters:
55
n_jobs: -2
66
dgp_parameters:
77
DGP:
8-
- 1
8+
- 2
9+
- 3
910
- 4
1011
n_obs:
11-
- 2000
12+
- 500
13+
xi:
14+
- 0.2
1215
learner_definitions:
1316
lgbmr: &id001
1417
name: LGBM Regr.
@@ -26,5 +29,4 @@ dml_parameters:
2629
- true
2730
confidence_parameters:
2831
level:
29-
- 0.95
3032
- 0.9

‎results/did/did_pa_multi_tune_detailed.csv‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
Learner g,Learner m,Score,Control Group,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,False,0.9120833333333332,1.1281458608527684,0.27675088039547086,0.94,1.766959072388302,3.597108107523503,2.9403917178509804,0.8534357674606872,200
3-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,True,0.89375,0.6376532313206807,0.15286697800217827,0.915,0.9983993286704501,3.29374008288805,2.748707897836178,0.679591632802887,200
4-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,False,0.9620833333333333,1.3442687059500862,0.27675088039547086,0.97,1.9367883375747112,3.597108107523503,2.9403917178509804,0.8534357674606872,200
5-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,True,0.9541666666666667,0.759810689252895,0.15286697800217827,0.945,1.0938791918349722,3.29374008288805,2.748707897836178,0.679591632802887,200
6-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.9041666666666667,1.4389904260432933,0.3468462138226047,0.91,2.2315687188052546,2.979241703960451,2.9994404795169123,0.8446712058892714,200
7-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.7629166666666667,0.7775230352607071,0.2683054004360395,0.63,1.2089929988521952,2.989557883560226,3.044965814893618,0.6749671427573682,200
8-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,False,0.9554166666666667,1.7146628508033268,0.3468462138226047,0.97,2.451356071426852,2.979241703960451,2.9994404795169123,0.8446712058892714,200
9-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,True,0.8429166666666668,0.926475840337015,0.2683054004360395,0.775,1.3270667190666499,2.989557883560226,3.044965814893618,0.6749671427573682,200
2+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,2,0.9,False,0.9266666666666667,3.0908848408903418,0.7161526833648638,0.96,4.742031668154074,5.382034194123735,5.231751941184026,0.8576041329426337,200
3+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,2,0.9,True,0.90125,1.8794931781846032,0.4591493790044287,0.865,2.8579990686983887,4.515059107298566,4.385233448105733,0.6919478275690014,200
4+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,3,0.9,False,0.9308333333333333,2.947927939580188,0.6723441057011548,0.96,4.521838848419626,4.642710293777947,4.646394978665751,0.8553041170789771,200
5+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,3,0.9,True,0.905,1.8011025121556412,0.4236148425151305,0.885,2.7558988253099552,4.228587388499047,4.190294595950896,0.6911269926300146,200
6+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.925,2.9018541428866937,0.6722860434862635,0.94,4.462288809787092,4.612699034697736,4.644364135602212,0.8550565538841952,200
7+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.8904166666666667,1.7864285303816843,0.4480420728291524,0.88,2.7393757933996197,4.181116753485783,4.208383450296873,0.6911607404321352,200

‎results/did/did_pa_multi_tune_eventstudy.csv‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
Learner g,Learner m,Score,Control Group,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,False,0.9075,1.044931024932891,0.24865990464197016,0.935,1.4671727358061626,3.597108107523503,2.9403917178509804,0.8534357674606872,200
3-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,True,0.8933333333333333,0.5855097565227854,0.13924209096865323,0.925,0.8214814724321132,3.29374008288805,2.748707897836178,0.679591632802887,200
4-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,False,0.9583333333333333,1.245112113101973,0.24865990464197016,0.985,1.6330868182303546,3.597108107523503,2.9403917178509804,0.8534357674606872,200
5-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,True,0.9508333333333333,0.6976779067620538,0.13924209096865323,0.945,0.9148690368152829,3.29374008288805,2.748707897836178,0.679591632802887,200
6-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.92,1.4033187650033796,0.3275689371006418,0.94,1.937780599061532,2.979241703960451,2.9994404795169123,0.8446712058892714,200
7-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.705,0.7446092869675938,0.27966561432340287,0.6,1.0285584537514965,2.989557883560226,3.044965814893618,0.6749671427573682,200
8-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,False,0.9683333333333333,1.6721574449961667,0.3275689371006418,0.975,2.1663251795251472,2.979241703960451,2.9994404795169123,0.8446712058892714,200
9-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,True,0.7883333333333333,0.8872566902596436,0.27966561432340287,0.745,1.153279573853566,2.989557883560226,3.044965814893618,0.6749671427573682,200
2+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,2,0.9,False,0.9366666666666668,2.97826761008284,0.6630614190615611,0.955,4.076327390258402,5.382034194123735,5.231751941184026,0.8576041329426337,200
3+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,2,0.9,True,0.9075,1.832594873095919,0.44252537334483505,0.895,2.4660898168275436,4.515059107298566,4.385233448105733,0.6919478275690014,200
4+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,3,0.9,False,0.9466666666666668,2.83754372010801,0.608791167628598,0.97,3.8789640516448474,4.642710293777947,4.646394978665751,0.8553041170789771,200
5+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,3,0.9,True,0.9225,1.7364545156585356,0.3998347112567276,0.91,2.3527693088573516,4.228587388499047,4.190294595950896,0.6911269926300146,200
6+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.925,2.81162753941394,0.6445514974670188,0.93,3.8526040538084434,4.612699034697736,4.644364135602212,0.8550565538841952,200
7+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.8891666666666667,1.7245938017850082,0.4367977362673907,0.845,2.342300467013995,4.181116753485783,4.208383450296873,0.6911607404321352,200

‎results/did/did_pa_multi_tune_group.csv‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
Learner g,Learner m,Score,Control Group,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,False,0.9116666666666667,1.0889543359860974,0.2591017535241754,0.91,1.3866236446708151,3.597108107523503,2.9403917178509804,0.8534357674606872,200
3-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,True,0.89,0.6229752808903669,0.15053905583073077,0.905,0.7933914451788466,3.29374008288805,2.748707897836178,0.679591632802887,200
4-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,False,0.9566666666666667,1.2975691237020013,0.2591017535241754,0.97,1.5696656784049068,3.597108107523503,2.9403917178509804,0.8534357674606872,200
5-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,True,0.9533333333333333,0.7423208325635811,0.15053905583073077,0.96,0.8996786017154156,3.29374008288805,2.748707897836178,0.679591632802887,200
6-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.9133333333333333,1.4454910752551433,0.3253154503600127,0.96,1.8286646948756324,2.979241703960451,2.9994404795169123,0.8446712058892714,200
7-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.74,0.7892476267585741,0.28546431334778133,0.645,1.0046266703621765,2.989557883560226,3.044965814893618,0.6749671427573682,200
8-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,False,0.9616666666666667,1.7224088521025238,0.3253154503600127,0.99,2.07498069683797,2.979241703960451,2.9994404795169123,0.8446712058892714,200
9-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,True,0.825,0.9404465528020834,0.28546431334778133,0.77,1.139020606132637,2.989557883560226,3.044965814893618,0.6749671427573682,200
2+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,2,0.9,False,0.95,3.1884243028188433,0.6875647807747396,0.965,4.048939927801238,5.382034194123735,5.231751941184026,0.8576041329426337,200
3+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,2,0.9,True,0.9066666666666667,1.9778194633199184,0.4796319143787132,0.9,2.49805265701074,4.515059107298566,4.385233448105733,0.6919478275690014,200
4+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,3,0.9,False,0.9366666666666668,3.0424311866970903,0.6450309879669672,0.95,3.8527728896927793,4.642710293777947,4.646394978665751,0.8553041170789771,200
5+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,3,0.9,True,0.905,1.8776034902629817,0.43604823577765456,0.9,2.3803383019346724,4.228587388499047,4.190294595950896,0.6911269926300146,200
6+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.94,2.959079715846458,0.6564246498507638,0.94,3.750120844819083,4.612699034697736,4.644364135602212,0.8550565538841952,200
7+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.895,1.8657876384470495,0.46631022079997453,0.86,2.364566875339995,4.181116753485783,4.208383450296873,0.6911607404321352,200

‎results/did/did_pa_multi_tune_metadata.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File
2-
0.12.dev0,DIDMultiTuningCoverageSimulation,2025-12-01 21:52,119.05648648738861,3.12.9,scripts/did/did_pa_multi_tune_config.yml
2+
0.12.dev0,DIDMultiTuningCoverageSimulation,2025-12-02 19:36,79.50826093753179,3.12.9,scripts/did/did_pa_multi_tune_config.yml

‎results/did/did_pa_multi_tune_time.csv‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
Learner g,Learner m,Score,Control Group,In-sample-norm.,DGP,level,Tuned,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,Loss g_control,Loss g_treated,Loss m,repetition
2-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,False,0.9433333333333332,1.0881198449012723,0.25682450851563937,0.945,1.3552240711392805,3.597108107523503,2.9403917178509804,0.8534357674606872,200
3-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.9,True,0.8966666666666667,0.5957046590186345,0.14226747638646786,0.915,0.7443420810052485,3.29374008288805,2.748707897836178,0.679591632802887,200
4-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,False,0.9716666666666667,1.2965747662437588,0.25682450851563937,0.96,1.5447661653225075,3.597108107523503,2.9403917178509804,0.8534357674606872,200
5-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,1,0.95,True,0.9533333333333333,0.7098258823571802,0.14226747638646786,0.955,0.8483239350473682,3.29374008288805,2.748707897836178,0.679591632802887,200
6-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.9183333333333333,1.4966364514582033,0.3513992824628867,0.94,1.8382232879162173,2.979241703960451,2.9994404795169123,0.8446712058892714,200
7-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.6516666666666667,0.787550236381641,0.30937396820647195,0.6,0.9660154081700578,2.989557883560226,3.044965814893618,0.6749671427573682,200
8-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,False,0.9716666666666667,1.7833523267626594,0.3513992824628867,0.965,2.102235052471558,2.979241703960451,2.9994404795169123,0.8446712058892714,200
9-
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.95,True,0.775,0.9384239874187676,0.30937396820647195,0.705,1.1067012746087066,2.989557883560226,3.044965814893618,0.6749671427573682,200
2+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,2,0.9,False,0.94,2.8577555367641163,0.6133825342587731,0.96,3.492613481221878,5.382034194123735,5.231751941184026,0.8576041329426337,200
3+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,2,0.9,True,0.9016666666666667,1.7948773086487813,0.4365123984827818,0.915,2.1646873101396342,4.515059107298566,4.385233448105733,0.6919478275690014,200
4+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,3,0.9,False,0.9466666666666668,2.759668649688763,0.5928088522386004,0.95,3.3907351416744462,4.642710293777947,4.646394978665751,0.8553041170789771,200
5+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,3,0.9,True,0.9133333333333333,1.705904234575853,0.39417420205664483,0.92,2.07199694125542,4.228587388499047,4.190294595950896,0.6911269926300146,200
6+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,False,0.9066666666666667,2.714971017166016,0.6476594323973127,0.915,3.332849955839899,4.612699034697736,4.644364135602212,0.8550565538841952,200
7+
LGBM Regr.,LGBM Clas.,observational,never_treated,True,4,0.9,True,0.8666666666666667,1.6906675567822227,0.43751542835810414,0.835,2.0555730584579877,4.181116753485783,4.208383450296873,0.6911607404321352,200

‎scripts/did/did_pa_multi_tune_config.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@ simulation_parameters:
77
n_jobs: -2
88

99
dgp_parameters:
10-
DGP: [1, 4] # Different DGP specifications
11-
n_obs: [2000] # Sample size for each simulation (has to be a list)
10+
DGP: [2, 3, 4] # Different DGP specifications
11+
n_obs: [500] # Sample size for each simulation (has to be a list)
12+
xi: [0.2]
1213

1314
# Define reusable learner configurations
1415
learner_definitions:
@@ -34,4 +35,4 @@ dml_parameters:
3435
in_sample_normalization: [true]
3536

3637
confidence_parameters:
37-
level: [0.95, 0.90] # Confidence levels
38+
level: [0.90] # Confidence levels

0 commit comments

Comments
 (0)